Sapian
/
kiwix-tools
forked from kiwix/kiwix-tools
3
0
Fork 0
Code Releases Activity

Merge pull request #229 from kiwix/new_kiwix-lib_api

Browse Source 
WIP New kiwix lib api
fix-bug-pid
Matthieu Gautier 7 years ago committed by GitHub
parent 881fe9f41d c1e635bd4e
commit eba80b92ef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 205 additions and 140 deletions
Show Stats Download Patch File Download Diff File

182
src/manager/kiwix-manage.cpp
Unescape View File

@ -22,33 +22,39 @@
#endif
#include <getopt.h>
#include <kiwix/common/pathTools.h>
#include <kiwix/common/stringTools.h>
#include <kiwix/manager.h>
#include <kiwix/downloader.h>
#include <cstdlib>
#include <iostream>
#include <thread>
#include <chrono>
using namespace std;
enum supportedAction { NONE, ADD, SHOW, REMOVE };
enum supportedAction { NONE, ADD, SHOW, REMOVE, DOWNLOAD };
void show(kiwix::Library library)
void show(kiwix::Library* library)
{
std::vector<kiwix::Book>::iterator itr;
auto booksIds = library->getBooksIds();
unsigned int inc = 1;
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
for(auto& id: booksIds) {
auto& book = library->getBookById(id);
std::cout << "#" << inc++ << std::endl
<< "id:\t\t" << itr->id << std::endl
<< "path:\t\t" << itr->path << std::endl
<< "indexpath:\t" << itr->indexPath << std::endl
<< "url:\t\t" << itr->url << std::endl
<< "title:\t\t" << itr->title << std::endl
<< "name:\t\t" << itr->name << std::endl
<< "tags:\t\t" << itr->tags << std::endl
<< "description:\t" << itr->description << std::endl
<< "creator:\t" << itr->creator << std::endl
<< "date:\t\t" << itr->date << std::endl
<< "articleCount:\t" << itr->articleCount << std::endl
<< "mediaCount:\t" << itr->mediaCount << std::endl
<< "size:\t\t" << itr->size << " KB" << std::endl
<< "id:\t\t" << book.getId() << std::endl
<< "path:\t\t" << book.getPath() << std::endl
<< "indexpath:\t" << book.getIndexPath() << std::endl
<< "url:\t\t" << book.getUrl() << std::endl
<< "title:\t\t" << book.getTitle() << std::endl
<< "name:\t\t" << book.getName() << std::endl
<< "tags:\t\t" << book.getTags() << std::endl
<< "description:\t" << book.getDescription() << std::endl
<< "creator:\t" << book.getCreator() << std::endl
<< "date:\t\t" << book.getDate() << std::endl
<< "articleCount:\t" << book.getArticleCount() << std::endl
<< "mediaCount:\t" << book.getMediaCount() << std::endl
<< "size:\t\t" << book.getSize() << " KB" << std::endl
<< std::endl;
}
}
@ -68,23 +74,21 @@ void usage()
}
bool handle_show(kiwix::Manager* libraryManager, const std::string& libraryPath,
bool handle_show(kiwix::Library* library, const std::string& libraryPath,
int argc, char* argv[])
{
show(libraryManager->cloneLibrary());
show(library);
return(0);
}
bool handle_add(kiwix::Manager* libraryManager, const std::string& libraryPath,
bool handle_add(kiwix::Library* library, const std::string& libraryPath,
int argc, char* argv[])
{
string zimPath;
string zimPathToSave = ".";
string indexPath;
kiwix::supportedIndexType indexBackend = kiwix::XAPIAN;
string url;
string origID = "";
bool setCurrent = false;
int option_index = 0;
int c = 0;
bool resultCode = 0;
@ -100,12 +104,10 @@ bool handle_add(kiwix::Manager* libraryManager, const std::string& libraryPath,
= {{"url", required_argument, 0, 'u'},
{"origId", required_argument, 0, 'o'},
{"indexPath", required_argument, 0, 'i'},
{"indexBackend", required_argument, 0, 'b'},
{"zimPathToSave", required_argument, 0, 'z'},
{"current", no_argument, 0, 'c'},
{0, 0, 0, 0}};
c = getopt_long(argc, argv, "cz:u:i:b:", long_options, &option_index);
c = getopt_long(argc, argv, "cz:u:i:", long_options, &option_index);
if (c != -1) {
switch (c) {
@ -117,22 +119,10 @@ bool handle_add(kiwix::Manager* libraryManager, const std::string& libraryPath,
origID = optarg;
break;
case 'c':
setCurrent = true;
break;
case 'i':
indexPath = optarg;
break;
case 'b':
if (!strcmp(optarg, "xapian")) {
indexBackend = kiwix::XAPIAN;
} else {
usage();
}
break;
case 'z':
zimPathToSave = optarg;
break;
@ -143,15 +133,18 @@ bool handle_add(kiwix::Manager* libraryManager, const std::string& libraryPath,
}
if (!zimPath.empty()) {
kiwix::Manager manager(library);
zimPathToSave = zimPathToSave == "." ? zimPath : zimPathToSave;
string bookId = libraryManager->addBookFromPathAndGetId(
string bookId = manager.addBookFromPathAndGetId(
zimPath, zimPathToSave, url, false);
if (!bookId.empty()) {
if (setCurrent)
libraryManager->setCurrentBookId(bookId);
/* Save the index infos if necessary */
if (!indexPath.empty())
libraryManager->setBookIndex(bookId, indexPath, indexBackend);
if (!indexPath.empty()) {
if (isRelativePath(indexPath)) {
indexPath = computeAbsolutePath(indexPath, getCurrentDirectory());
}
library->getBookById(bookId).setIndexPath(indexPath);
}
} else {
cerr << "Unable to build or save library file '" << libraryPath << "'"
<< endl;
@ -165,28 +158,25 @@ bool handle_add(kiwix::Manager* libraryManager, const std::string& libraryPath,
return(resultCode);
}
bool handle_remove(kiwix::Manager* libraryManager, const std::string& libraryPath,
bool handle_remove(kiwix::Library* library, const std::string& libraryPath,
int argc, char* argv[])
{
unsigned int bookIndex = 0;
const unsigned int totalBookCount = libraryManager->getBookCount(true, true);
std::string bookId;
const unsigned int totalBookCount = library->getBookCount(true, true);
bool exitCode = 0;
if (argc > 3) {
bookIndex = atoi(argv[3]);
bookId = argv[3];
}
if (bookIndex > 0 && bookIndex <= totalBookCount) {
libraryManager->removeBookByIndex(bookIndex - 1);
} else {
if (!library->removeBookById(bookId)) {
if (totalBookCount > 0) {
std::cerr
<< "Invalid book index number. Please give a number between 1 and "
<< totalBookCount << std::endl;
<< "Invalid book id." << std::endl;
exitCode = 1;
} else {
std::cerr
<< "Invalid book index number. Library is empty, no book to delete."
<< "Invalid book id. Library is empty, no book to delete."
<< std::endl;
exitCode = 1;
}
@ -195,11 +185,67 @@ bool handle_remove(kiwix::Manager* libraryManager, const std::string& libraryPat
return(exitCode);
}
bool handle_download(kiwix::Library* library, const std::string& libraryPath,
int argc, char* argv[])
{
std::string bookId;
bool exitCode = false;
if (argc > 3) {
bookId = argv[3];
}
auto& book = library->getBookById(bookId);
auto did = book.getDownloadId();
kiwix::Downloader downloader;
kiwix::Download* download = nullptr;
if (!did.empty()) {
std::cout << "try resume " << did << std::endl;
try {
download = downloader.getDownload(did);
} catch(...) {}
}
if (nullptr == download || download->getStatus() == kiwix::Download::K_UNKNOWN) {
download = downloader.startDownload(book.getUrl());
book.setDownloadId(download->getDid());
}
int step = 60*5;
while (step--) {
download->updateStatus();
if (download->getStatus() == kiwix::Download::K_COMPLETE) {
auto followingId = download->getFollowedBy();
if (followingId.empty()) {
book.setPath(download->getPath());
book.setDownloadId("");
std::cout << "File downloaded to " << book.getPath() << std::endl;
break;
} else {
download = downloader.getDownload(followingId);
}
} else if (download->getStatus() == kiwix::Download::K_ACTIVE) {
std::cout << download->getDid() << " : "
<< kiwix::beautifyFileSize(download->getCompletedLength()) << "/"
<< kiwix::beautifyFileSize(download->getTotalLength())
<< " (" << kiwix::beautifyFileSize(download->getDownloadSpeed()) << "/s) "
<< " [" << kiwix::beautifyFileSize(download->getVerifiedLength()) << "]"
<< "[" << step << "] \n" << std::flush;
} else if (download->getStatus() == kiwix::Download::K_ERROR) {
std::cout << "File Error" << std::endl;
exitCode = true;
break;
}
std::this_thread::sleep_for(std::chrono::seconds(1));
}
downloader.close();
return exitCode;
}
int main(int argc, char** argv)
{
string libraryPath = "";
supportedAction action = NONE;
kiwix::Manager libraryManager;
kiwix::Library library;
/* Argument parsing */
if (argc > 2) {
@ -212,6 +258,8 @@ int main(int argc, char** argv)
action = SHOW;
else if (actionString == "remove" || actionString == "delete")
action = REMOVE;
else if (actionString == "download")
action = DOWNLOAD;
}
/* Print usage)) if necessary */
@ -224,21 +272,31 @@ int main(int argc, char** argv)
libraryPath = isRelativePath(libraryPath)
? computeAbsolutePath(getCurrentDirectory(), libraryPath)
: libraryPath;
libraryManager.readFile(libraryPath, false);
kiwix::Manager manager(&library);
manager.readFile(libraryPath, false);
/* SHOW */
bool exitCode = 0;
if (action == SHOW) {
exitCode = handle_show(&libraryManager, libraryPath, argc, argv);
} else if (action == ADD) {
exitCode = handle_add(&libraryManager, libraryPath, argc, argv);
} else if (action == REMOVE) {
exitCode = handle_remove(&libraryManager, libraryPath, argc, argv);
switch (action) {
case SHOW:
exitCode = handle_show(&library, libraryPath, argc, argv);
break;
case ADD:
exitCode = handle_add(&library, libraryPath, argc, argv);
break;
case REMOVE:
exitCode = handle_remove(&library, libraryPath, argc, argv);
break;
case DOWNLOAD:
exitCode = handle_download(&library, libraryPath, argc, argv);
break;
case NONE:
break;
}
/* Rewrite the library file */
if (action == REMOVE || action == ADD) {
libraryManager.writeFile(libraryPath);
if (action == REMOVE || action == ADD || action == DOWNLOAD) {
library.writeToFile(libraryPath);
}
exit(exitCode);

160
src/server/kiwix-serve.cpp
Unescape View File

@ -101,20 +101,11 @@ static std::map<std::string, std::string> extMimeTypes;
static std::map<std::string, kiwix::Reader*> readers;
static std::map<std::string, kiwix::Searcher*> searchers;
static kiwix::Searcher* globalSearcher = nullptr;
static kiwix::Manager libraryManager;
static kiwix::Library library;
static pthread_mutex_t searchLock = PTHREAD_MUTEX_INITIALIZER;
static pthread_mutex_t compressorLock = PTHREAD_MUTEX_INITIALIZER;
static pthread_mutex_t regexLock = PTHREAD_MUTEX_INITIALIZER;
template<typename T>
inline std::string _tostring(const T& value)
{
std::ostringstream stream;
stream << value;
return stream.str();
}
std::pair<kiwix::Reader*, kiwix::Searcher*>
get_from_humanReadableBookId(const std::string& humanReadableBookId) {
kiwix::Searcher* searcher
@ -385,7 +376,7 @@ static struct MHD_Response* build_callback_response_from_entry(
MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_RANGE, oss.str().c_str());
MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_LENGTH,
_tostring(range_len).c_str());
kiwix::to_string(range_len).c_str());
/* Specify the mime type */
MHD_add_response_header(
@ -694,33 +685,51 @@ static struct MHD_Response* handle_catalog(RequestContext* request)
kiwix::OPDSDumper opdsDumper;
opdsDumper.setRootLocation(rootLocation);
opdsDumper.setSearchDescriptionUrl("catalog/searchdescription.xml");
opdsDumper.setId(kiwix::to_string(uuid));
opdsDumper.setLibrary(&library);
mimeType = "application/atom+xml;profile=opds-catalog;kind=acquisition; charset=utf-8";
kiwix::Library library_to_dump;
std::vector<std::string> bookIdsToDump;
if (url == "root.xml") {
opdsDumper.setTitle("All zims");
uuid = zim::Uuid::generate(host);
library_to_dump = libraryManager.cloneLibrary();
bookIdsToDump = library.listBooksIds(
kiwix::VALID|kiwix::LOCAL|kiwix::REMOTE);
} else if (url == "search") {
std::string query;
std::string language;
size_t count(10);
size_t startIndex(0);
try {
query = request->get_argument("q");
} catch (const std::out_of_range&) {
return build_404(request, "");
}
} catch (const std::out_of_range&) {}
try {
language = request->get_argument("lang");
} catch (const std::out_of_range&) {}
try {
count = stoul(request->get_argument("count"));
} catch (...) {}
try {
startIndex = stoul(request->get_argument("start"));
} catch (...) {}
opdsDumper.setTitle("Search result for " + query);
uuid = zim::Uuid::generate();
library_to_dump = libraryManager.filter(query);
bookIdsToDump = library.listBooksIds(
kiwix::VALID|kiwix::LOCAL|kiwix::REMOTE,
kiwix::UNSORTED,
query,
language);
auto totalResults = bookIdsToDump.size();
bookIdsToDump.erase(bookIdsToDump.begin(), bookIdsToDump.begin()+startIndex);
if (count>0 && bookIdsToDump.size() > count) {
bookIdsToDump.resize(count);
}
opdsDumper.setOpenSearchInfo(totalResults, startIndex, bookIdsToDump.size());
} else {
return build_404(request, "");
}
{
std::stringstream ss;
ss << uuid;
opdsDumper.setId(ss.str());
}
opdsDumper.setLibrary(library_to_dump);
content = opdsDumper.dumpOPDSFeed();
content = opdsDumper.dumpOPDSFeed(bookIdsToDump);
}
bool deflated = request->can_compress() && compress_content(content, mimeType);
@ -1021,6 +1030,7 @@ int main(int argc, char** argv)
}
/* Setup the library manager and get the list of books */
kiwix::Manager manager(&library);
if (libraryFlag) {
vector<string> libraryPaths = kiwix::split(libraryPath, ";");
vector<string>::iterator itr;
@ -1034,7 +1044,7 @@ int main(int argc, char** argv)
= isRelativePath(*itr)
? computeAbsolutePath(getCurrentDirectory(), *itr)
: *itr;
retVal = libraryManager.readFile(libraryPath, true);
retVal = manager.readFile(libraryPath, true);
} catch (...) {
retVal = false;
}
@ -1048,74 +1058,69 @@ int main(int argc, char** argv)
}
/* Check if the library is not empty (or only remote books)*/
if (libraryManager.getBookCount(true, false) == 0) {
if (library.getBookCount(true, false) == 0) {
cerr << "The XML library file '" << libraryPath
<< "' is empty (or has only remote books)." << endl;
}
} else {
std::vector<std::string>::iterator it;
for (it = zimPathes.begin(); it != zimPathes.end(); it++) {
if (!libraryManager.addBookFromPath(*it, *it, "", false)) {
if (!manager.addBookFromPath(*it, *it, "", false)) {
cerr << "Unable to add the ZIM file '" << *it
<< "' to the internal library." << endl;
exit(1);
}
}
if (!indexPath.empty()) {
libraryManager.setBookIndex(libraryManager.getBooksIds()[0], indexPath);
if (isRelativePath(indexPath)) {
indexPath = computeAbsolutePath(indexPath, getCurrentDirectory());
}
library.getBookById(library.getBooksIds()[0]).setIndexPath(indexPath);
}
}
/* Instance the readers and searcher and build the corresponding maps */
vector<string> booksIds = libraryManager.getBooksIds();
vector<string>::iterator itr;
kiwix::Book currentBook;
vector<string> booksIds = library.listBooksIds(kiwix::LOCAL);
globalSearcher = new kiwix::Searcher();
globalSearcher->setProtocolPrefix(rootLocation + "/");
globalSearcher->setSearchProtocolPrefix(rootLocation + "/" + "search?");
for (itr = booksIds.begin(); itr != booksIds.end(); ++itr) {
bool zimFileOk = false;
libraryManager.getBookById(*itr, currentBook);
std::string zimPath = currentBook.pathAbsolute;
for (auto& bookId: booksIds) {
auto& currentBook = library.getBookById(bookId);
auto zimPath = currentBook.getPath();
auto indexPath = currentBook.getIndexPath();
if (!zimPath.empty()) {
indexPath = currentBook.indexPathAbsolute;
/* Instanciate the ZIM file handler */
kiwix::Reader* reader = NULL;
try {
reader = new kiwix::Reader(zimPath);
} catch (...) {
cerr << "Unable to open the ZIM file '" << zimPath << "'." << endl;
continue;
}
/* Instanciate the ZIM file handler */
kiwix::Reader* reader = NULL;
auto humanReadableId = currentBook.getHumanReadableIdFromPath();
readers[humanReadableId] = reader;
if (reader->hasFulltextIndex()) {
kiwix::Searcher* searcher = new kiwix::Searcher(humanReadableId);
searcher->setProtocolPrefix(rootLocation + "/");
searcher->setSearchProtocolPrefix(rootLocation + "/" + "search?");
searcher->add_reader(reader, humanReadableId);
globalSearcher->add_reader(reader, humanReadableId);
searchers[humanReadableId] = searcher;
} else if ( !indexPath.empty() ) {
try {
reader = new kiwix::Reader(zimPath);
zimFileOk = true;
kiwix::Searcher* searcher = new kiwix::Searcher(indexPath, reader, humanReadableId);
searcher->setProtocolPrefix(rootLocation + "/");
searcher->setSearchProtocolPrefix(rootLocation + "/" + "search?");
searchers[humanReadableId] = searcher;
} catch (...) {
cerr << "Unable to open the ZIM file '" << zimPath << "'." << endl;
}
if (zimFileOk) {
string humanReadableId = currentBook.getHumanReadableIdFromPath();
readers[humanReadableId] = reader;
if ( reader->hasFulltextIndex()) {
kiwix::Searcher* searcher = new kiwix::Searcher(humanReadableId);
searcher->setProtocolPrefix(rootLocation + "/");
searcher->setSearchProtocolPrefix(rootLocation + "/" + "search?");
searcher->add_reader(reader, humanReadableId);
globalSearcher->add_reader(reader, humanReadableId);
searchers[humanReadableId] = searcher;
} else if ( !indexPath.empty() ) {
try {
kiwix::Searcher* searcher = new kiwix::Searcher(indexPath, reader, humanReadableId);
searcher->setProtocolPrefix(rootLocation + "/");
searcher->setSearchProtocolPrefix(rootLocation + "/" + "search?");
searchers[humanReadableId] = searcher;
} catch (...) {
cerr << "Unable to open the search index '" << indexPath << "'."
<< endl;
searchers[humanReadableId] = nullptr;
}
} else {
searchers[humanReadableId] = nullptr;
}
cerr << "Unable to open the search index '" << indexPath << "'."
<< endl;
searchers[humanReadableId] = nullptr;
}
} else {
searchers[humanReadableId] = nullptr;
}
}
@ -1123,24 +1128,25 @@ int main(int argc, char** argv)
string welcomeBooksHtml
= ""
"<div class='book__list'>";
for (itr = booksIds.begin(); itr != booksIds.end(); ++itr) {
libraryManager.getBookById(*itr, currentBook);
for (auto& bookId: booksIds) {
auto& currentBook = library.getBookById(bookId);
if (!currentBook.path.empty()
if (!currentBook.getPath().empty()
&& readers.find(currentBook.getHumanReadableIdFromPath())
!= readers.end()) {
welcomeBooksHtml += ""
"<a href='" + rootLocation + "/" + currentBook.getHumanReadableIdFromPath() + "/'>"
"<div class='book'>"
"<div class='book__background' style='background-image: url(data:" + currentBook.faviconMimeType+ ";base64," + currentBook.favicon + ");'>"
"<div class='book__title' title='" + currentBook.title + "'>" + currentBook.title + "</div>"
"<div class='book__description' title='" + currentBook.description + "'>" + currentBook.description + "</div>"
"<div class='book__background' style=\"background-image: url('/meta?content="
+ currentBook.getHumanReadableIdFromPath() + "&name=favicon');\">"
"<div class='book__title' title='" + currentBook.getTitle() + "'>" + currentBook.getTitle() + "</div>"
"<div class='book__description' title='" + currentBook.getDescription() + "'>" + currentBook.getDescription() + "</div>"
"<div class='book__info'>"
"" + kiwix::beautifyInteger(atoi(currentBook.articleCount.c_str())) + " articles, " + kiwix::beautifyInteger(atoi(currentBook.mediaCount.c_str())) + " medias"
"" + kiwix::beautifyInteger(currentBook.getArticleCount()) + " articles, " + kiwix::beautifyInteger(currentBook.getMediaCount()) + " medias"
"</div>"
"</div>"
"</div>"
"</a>";
"</a>\n";
}
}
welcomeBooksHtml += ""

3
static/server/opensearchdescription.xml
Unescape View File

@ -4,5 +4,6 @@
<Description>Search zim files in the catalog.</Description>
<Url type="application/atom+xml;profile=opds-catalog"
xmlns:atom="http://www.w3.org/2005/Atom"
template="/__ROOT_LOCATION__/catalog/search?q={searchTerms}"/>
indexOffset="0"
template="/__ROOT_LOCATION__/catalog/search?q={searchTerms}&lang={language}&count={count}&start={startIndex}"/>
</OpenSearchDescription>

Write Preview
Loading…
Cancel
Save