Sapian
/
kiwix-tools
forked from kiwix/kiwix-tools
3
0
Fork 0
Code Releases Activity

Format all the code using clang-format.

Browse Source 
Add a script `format_code.sh` to easily format the code.
small_fixes
Matthieu Gautier 8 years ago
parent 856bfc675a
commit 4e3ff03059
7 changed files with 705 additions and 570 deletions
Show Stats Download Patch File Download Diff File

12
.clang-format
Unescape View File

@ -0,0 +1,12 @@
BasedOnStyle: Google
BinPackArguments: false
BinPackParameters: false
BreakBeforeBinaryOperators: All
BreakBeforeBraces: Linux
DerivePointerAlignment: false
SpacesInContainerLiterals: false
Standard: Cpp11
AllowShortFunctionsOnASingleLine: Inline
AllowShortIfStatementsOnASingleLine: false
AllowShortLoopsOnASingleLine: false

15
format_code.sh
Unescape View File

@ -0,0 +1,15 @@
#!/usr/bin/bash
files=(
"src/installer/kiwix-install.cpp"
"src/searcher/kiwix-search.cpp"
"src/reader/kiwix-read.cpp"
"src/manager/kiwix-manage.cpp"
"src/server/kiwix-serve.cpp"
)
for i in "${files[@]}"
do
echo $i
clang-format -i -style=file $i
done

68
src/installer/kiwix-install.cpp
Unescape View File

@ -19,18 +19,20 @@
#include <getopt.h>
#include <kiwix/common/pathTools.h>
#include <kiwix/reader.h>
#include <kiwix/manager.h>
#include <kiwix/reader.h>
enum supportedAction { NONE, ADDCONTENT };
void usage() {
cout << "Usage: kiwix-install [--verbose] addcontent ZIM_PATH KIWIX_PATH" << endl;
void usage()
{
cout << "Usage: kiwix-install [--verbose] addcontent ZIM_PATH KIWIX_PATH"
<< endl;
exit(1);
}
int main(int argc, char **argv) {
int main(int argc, char** argv)
{
/* Init the variables */
const char* contentPath = NULL;
const char* kiwixPath = NULL;
@ -41,11 +43,8 @@ int main(int argc, char **argv) {
/* Argument parsing */
while (42) {
static struct option long_options[] = {
{"verbose", no_argument, 0, 'v'},
{0, 0, 0, 0}
};
static struct option long_options[]
= {{"verbose", no_argument, 0, 'v'}, {0, 0, 0, 0}};
if (c != -1) {
c = getopt_long(argc, argv, "vi", long_options, &option_index);
@ -82,38 +81,48 @@ int main(int argc, char **argv) {
/* Make the action */
if (action == ADDCONTENT) {
/* Check if the content path exists and is readable */
if (verboseFlag) { std::cout << "Check if the ZIM file exists..." << std::endl; }
if (verboseFlag) {
std::cout << "Check if the ZIM file exists..." << std::endl;
}
if (!fileExists(contentPath)) {
cerr << "The content path '" << contentPath << "' does not exist or is not readable." << endl;
cerr << "The content path '" << contentPath
<< "' does not exist or is not readable." << endl;
exit(1);
}
/* Check if this is a ZIM file */
try {
if (verboseFlag) { std::cout << "Check if the ZIM file is valid..." << std::endl; }
if (verboseFlag) {
std::cout << "Check if the ZIM file is valid..." << std::endl;
}
kiwix::Reader* reader = new kiwix::Reader(contentPath);
delete reader;
} catch (exception& e) {
cerr << "The content available at '" << contentPath << "' is not a ZIM file." << endl;
cerr << "The content available at '" << contentPath
<< "' is not a ZIM file." << endl;
exit(1);
}
string contentFilename = getLastPathElement(contentPath);
/* Check if kiwixPath/kiwix/kiwix.exe exists */
if (verboseFlag) { std::cout << "Check if the Kiwix path is valid..." << std::endl; }
if (verboseFlag) {
std::cout << "Check if the Kiwix path is valid..." << std::endl;
}
string kiwixBinaryPath = computeAbsolutePath(kiwixPath, "kiwix/kiwix.exe");
if (!fileExists(kiwixBinaryPath)) {
kiwixBinaryPath = computeAbsolutePath(kiwixPath, "kiwix/kiwix");
if (!fileExists(kiwixBinaryPath)) {
cerr << "Unable to find the Kiwix binary at '" << kiwixBinaryPath << "[.exe]'." << endl;
cerr << "Unable to find the Kiwix binary at '" << kiwixBinaryPath
<< "[.exe]'." << endl;
exit(1);
}
}
/* Check if the directory "data" structure exists */
if (verboseFlag) { std::cout << "Check the target data directory structure..." << std::endl; }
if (verboseFlag) {
std::cout << "Check the target data directory structure..." << std::endl;
}
string dataPath = computeAbsolutePath(kiwixPath, "data/");
if (!fileExists(dataPath)) {
makeDirectory(dataPath);
@ -132,22 +141,31 @@ int main(int argc, char **argv) {
}
/* Copy the file to the data/content directory */
if (verboseFlag) { std::cout << "Copy ZIM file to the target directory..." << std::endl; }
string newContentPath = computeAbsolutePath(dataContentPath, contentFilename);
if (!fileExists(newContentPath) || getFileSize(contentPath) != getFileSize(newContentPath)) {
if (verboseFlag) {
std::cout << "Copy ZIM file to the target directory..." << std::endl;
}
string newContentPath
= computeAbsolutePath(dataContentPath, contentFilename);
if (!fileExists(newContentPath)
|| getFileSize(contentPath) != getFileSize(newContentPath)) {
copyFile(contentPath, newContentPath);
}
/* Add the file to the library.xml */
if (verboseFlag) { std::cout << "Create the library XML file..." << std::endl; }
if (verboseFlag) {
std::cout << "Create the library XML file..." << std::endl;
}
kiwix::Manager libraryManager;
string libraryPath = computeAbsolutePath(dataLibraryPath, contentFilename + ".xml");
string bookId = libraryManager.addBookFromPathAndGetId(newContentPath, "../content/" + contentFilename, "", false);
string libraryPath
= computeAbsolutePath(dataLibraryPath, contentFilename + ".xml");
string bookId = libraryManager.addBookFromPathAndGetId(
newContentPath, "../content/" + contentFilename, "", false);
if (!bookId.empty()) {
libraryManager.setCurrentBookId(bookId);
libraryManager.writeFile(libraryPath);
} else {
cerr << "Unable to build or save library file '" << libraryPath << "'" << endl;
cerr << "Unable to build or save library file '" << libraryPath << "'"
<< endl;
}
}

92
src/manager/kiwix-manage.cpp
Unescape View File

@ -21,47 +21,54 @@
#include <unistd.h>
#endif
#include <getopt.h>
#include <iostream>
#include <cstdlib>
#include <kiwix/common/pathTools.h>
#include <kiwix/manager.h>
#include <cstdlib>
#include <iostream>
using namespace std;
enum supportedAction { NONE, ADD, SHOW, REMOVE };
void show(kiwix::Library library) {
void show(kiwix::Library library)
{
std::vector<kiwix::Book>::iterator itr;
unsigned int inc = 1;
for (itr = library.books.begin(); itr != library.books.end(); ++itr) {
std::cout << "#" << inc++
<< std::endl << "id:\t\t" << itr->id
<< std::endl << "path:\t\t" << itr->path
<< std::endl << "indexpath:\t" << itr->indexPath
<< std::endl << "url:\t\t" << itr->url
<< std::endl << "title:\t\t" << itr->title
<< std::endl << "name:\t\t" << itr->name
<< std::endl << "tags:\t\t" << itr->tags
<< std::endl << "description:\t" << itr->description
<< std::endl << "creator:\t" << itr->creator
<< std::endl << "date:\t\t" << itr->date
<< std::endl << "articleCount:\t" << itr->articleCount
<< std::endl << "mediaCount:\t" << itr->mediaCount
<< std::endl << "size:\t\t" << itr->size << " KB"
<< std::endl << std::endl;
}
}
void usage() {
std::cout << "#" << inc++ << std::endl
<< "id:\t\t" << itr->id << std::endl
<< "path:\t\t" << itr->path << std::endl
<< "indexpath:\t" << itr->indexPath << std::endl
<< "url:\t\t" << itr->url << std::endl
<< "title:\t\t" << itr->title << std::endl
<< "name:\t\t" << itr->name << std::endl
<< "tags:\t\t" << itr->tags << std::endl
<< "description:\t" << itr->description << std::endl
<< "creator:\t" << itr->creator << std::endl
<< "date:\t\t" << itr->date << std::endl
<< "articleCount:\t" << itr->articleCount << std::endl
<< "mediaCount:\t" << itr->mediaCount << std::endl
<< "size:\t\t" << itr->size << " KB" << std::endl
<< std::endl;
}
}
void usage()
{
cerr << "Usage:" << endl;
cerr << "\tkiwix-manage LIBRARY_PATH add ZIM_PATH [--zimPathToSave=../content/foobar.zim] [--current] [--indexBackend=xapian] [--indexPath=FULLTEXT_IDX_PATH] [--url=http://...metalink]" << endl;
cerr << "\tkiwix-manage LIBRARY_PATH show [CONTENTID1] [CONTENTID2] ... (show everything if no param.)" << endl;
cerr << "\tkiwix-manage LIBRARY_PATH add ZIM_PATH "
"[--zimPathToSave=../content/foobar.zim] [--current] "
"[--indexBackend=xapian] [--indexPath=FULLTEXT_IDX_PATH] "
"[--url=http://...metalink]"
<< endl;
cerr << "\tkiwix-manage LIBRARY_PATH show [CONTENTID1] [CONTENTID2] ... "
"(show everything if no param.)"
<< endl;
cerr << "\tkiwix-manage LIBRARY_PATH remove CONTENTID1 [CONTENTID2]" << endl;
}
int main(int argc, char **argv) {
int main(int argc, char** argv)
{
string libraryPath = "";
supportedAction action = NONE;
string zimPath = "";
@ -89,9 +96,9 @@ int main(int argc, char **argv) {
}
/* Try to read the file */
libraryPath = isRelativePath(libraryPath) ?
computeAbsolutePath(getCurrentDirectory(), libraryPath) :
libraryPath;
libraryPath = isRelativePath(libraryPath)
? computeAbsolutePath(getCurrentDirectory(), libraryPath)
: libraryPath;
libraryManager.readFile(libraryPath, false);
/* SHOW */
@ -113,21 +120,18 @@ int main(int argc, char **argv) {
/* Options parsing */
optind = 2;
while (42) {
static struct option long_options[] = {
{"url", required_argument, 0, 'u'},
static struct option long_options[]
= {{"url", required_argument, 0, 'u'},
{"origId", required_argument, 0, 'o'},
{"indexPath", required_argument, 0, 'i'},
{"indexBackend", required_argument, 0, 'b'},
{"zimPathToSave", required_argument, 0, 'z'},
{"current", no_argument, 0, 'c'},
{0, 0, 0, 0}
};
{0, 0, 0, 0}};
c = getopt_long(argc, argv, "cz:u:i:b:", long_options, &option_index);
if (c != -1) {
switch (c) {
case 'u':
url = optarg;
@ -156,7 +160,6 @@ int main(int argc, char **argv) {
case 'z':
zimPathToSave = optarg;
break;
}
} else {
break;
@ -165,10 +168,10 @@ int main(int argc, char **argv) {
if (!zimPath.empty()) {
zimPathToSave = zimPathToSave == "." ? zimPath : zimPathToSave;
string bookId = libraryManager.addBookFromPathAndGetId(zimPath, zimPathToSave, url, false);
string bookId = libraryManager.addBookFromPathAndGetId(
zimPath, zimPathToSave, url, false);
if (!bookId.empty()) {
if (setCurrent)
libraryManager.setCurrentBookId(bookId);
@ -177,7 +180,8 @@ int main(int argc, char **argv) {
libraryManager.setBookIndex(bookId, indexPath, indexBackend);
} else {
cerr << "Unable to build or save library file '" << libraryPath << "'" << endl;
cerr << "Unable to build or save library file '" << libraryPath << "'"
<< endl;
}
} else {
std::cerr << "Invalid zim file path" << std::endl;
@ -195,9 +199,13 @@ int main(int argc, char **argv) {
libraryManager.removeBookByIndex(bookIndex - 1);
} else {
if (totalBookCount > 0) {
std::cerr << "Invalid book index number. Please give a number between 1 and " << totalBookCount << std::endl;
std::cerr
<< "Invalid book index number. Please give a number between 1 and "
<< totalBookCount << std::endl;
} else {
std::cerr << "Invalid book index number. Library is empty, no book to delete." << std::endl;
std::cerr
<< "Invalid book index number. Library is empty, no book to delete."
<< std::endl;
}
}
}

28
src/reader/kiwix-read.cpp
Unescape View File

@ -18,23 +18,26 @@
*/
#include <getopt.h>
#include <unistd.h>
#include <string>
#include <map>
#include <kiwix/common/tree.h>
#include <kiwix/reader.h>
#include <unistd.h>
#include <map>
#include <string>
void usage() {
void usage()
{
cout << "Usage: kiwix-read --suggest=<PATTERN> ZIM_FILE_PATH" << endl;
exit(1);
}
void updateSuggestionTree(tree< pair<string, unsigned> > &suggestionTree, string suggestion) {
void updateSuggestionTree(tree<pair<string, unsigned>>& suggestionTree,
string suggestion)
{
return;
}
int main(int argc, char **argv) {
int main(int argc, char** argv)
{
/* Init the variables */
const char* filePath = NULL;
const char* pattern = NULL;
@ -45,12 +48,10 @@ int main(int argc, char **argv) {
/* Argument parsing */
while (42) {
static struct option long_options[] = {
{"verbose", no_argument, 0, 'v'},
static struct option long_options[]
= {{"verbose", no_argument, 0, 'v'},
{"suggest", required_argument, 0, 's'},
{0, 0, 0, 0}
};
{0, 0, 0, 0}};
if (c != -1) {
c = getopt_long(argc, argv, "vs:", long_options, &option_index);
@ -101,7 +102,8 @@ int main(int argc, char **argv) {
}
/*
if (reader->getContentByUrl(mainPageUrl, content, contentLength, contentType)) {
if (reader->getContentByUrl(mainPageUrl, content, contentLength,
contentType)) {
cout << content << endl;
}
*/

22
src/searcher/kiwix-search.cpp
Unescape View File

@ -18,19 +18,21 @@
*/
#include <getopt.h>
#include <unistd.h>
#include <kiwix/reader.h>
#include <kiwix/searcher.h>
#include <unistd.h>
void usage() {
void usage()
{
cout << "Usage: kiwix-search [--verbose|-v] ZIM_PATH SEARCH" << endl;
exit(1);
}
int main(int argc, char **argv) {
int main(int argc, char** argv)
{
/* Init the variables */
//const char *indexPath = "/home/itamar/.www.kiwix.org/kiwix/43k0i1j4.default/6d2e587b-d586-dc6a-dc6a-e4ef035a1495d15c.index";
// const char *indexPath =
// "/home/itamar/.www.kiwix.org/kiwix/43k0i1j4.default/6d2e587b-d586-dc6a-dc6a-e4ef035a1495d15c.index";
// const char *indexPath = "/home/itamar/testindex";
const char* zimPath = NULL;
const char* search = NULL;
@ -43,11 +45,8 @@ int main(int argc, char **argv) {
/* Argument parsing */
while (42) {
static struct option long_options[] = {
{"verbose", no_argument, 0, 'v'},
{0, 0, 0, 0}
};
static struct option long_options[]
= {{"verbose", no_argument, 0, 'v'}, {0, 0, 0, 0}};
if (c != -1) {
c = getopt_long(argc, argv, "vb:", long_options, &option_index);
@ -82,7 +81,8 @@ int main(int argc, char **argv) {
try {
reader = new kiwix::Reader(zimPath);
} catch (...) {
/* Cannot open the zimPath, maybe it is a plain old xapian database directory */
/* Cannot open the zimPath, maybe it is a plain old xapian database
* directory */
}
if (reader) {

394
src/server/kiwix-serve.cpp
Unescape View File

@ -20,8 +20,8 @@
#define KIWIX_MIN_CONTENT_SIZE_TO_DEFLATE 100
#ifdef __APPLE__
#import <sys/types.h>
#import <sys/sysctl.h>
#import <sys/types.h>
#define MIBSIZE 4
#endif
@ -43,40 +43,40 @@ extern "C" {
#endif
#include <getopt.h>
#include <kiwix/common/otherTools.h>
#include <kiwix/common/pathTools.h>
#include <kiwix/common/regexTools.h>
#include <kiwix/common/stringTools.h>
#include <kiwix/manager.h>
#include <kiwix/reader.h>
#include <kiwix/searcher.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <getopt.h>
#include <iostream>
#include <string>
#include <vector>
#include <map>
#include <fstream>
#include <iostream>
#include <sstream>
#include <zim/zim.h>
#include <zim/file.h>
#include <zim/article.h>
#include <zim/file.h>
#include <zim/fileiterator.h>
#include <pthread.h>
#include <zim/zim.h>
#include <zlib.h>
#include <kiwix/reader.h>
#include <kiwix/manager.h>
#include <kiwix/searcher.h>
#include <kiwix/common/pathTools.h>
#include <kiwix/common/regexTools.h>
#include <kiwix/common/stringTools.h>
#include <kiwix/common/otherTools.h>
#include <fstream>
#include <iostream>
#include <iostream>
#include <map>
#include <sstream>
#include <string>
#include <vector>
#include "server-resources.h"
#ifndef _WIN32
#include <stdint.h>
#include <unistd.h>
#include <microhttpd.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <netdb.h>
#include <ifaddrs.h>
#include <microhttpd.h>
#include <netdb.h>
#include <stdint.h>
#include <sys/socket.h>
#include <unistd.h>
#endif
#ifdef interface
@ -101,7 +101,8 @@ static pthread_mutex_t verboseFlagLock = PTHREAD_MUTEX_INITIALIZER;
static pthread_mutex_t mimeTypeLock = PTHREAD_MUTEX_INITIALIZER;
/* Try to get the mimeType from the file extension */
static std::string getMimeTypeForFile(const std::string& filename) {
static std::string getMimeTypeForFile(const std::string& filename)
{
std::string mimeType = "text/plain";
unsigned int pos = filename.find_last_of(".");
@ -111,7 +112,8 @@ static std::string getMimeTypeForFile(const std::string& filename) {
pthread_mutex_lock(&mimeTypeLock);
if (extMimeTypes.find(extension) != extMimeTypes.end()) {
mimeType = extMimeTypes[extension];
} else if (extMimeTypes.find(kiwix::lcAll(extension)) != extMimeTypes.end()) {
} else if (extMimeTypes.find(kiwix::lcAll(extension))
!= extMimeTypes.end()) {
mimeType = extMimeTypes[kiwix::lcAll(extension)];
}
pthread_mutex_unlock(&mimeTypeLock);
@ -120,25 +122,33 @@ static std::string getMimeTypeForFile(const std::string& filename) {
return mimeType;
}
void introduceTaskbar(string &content, const string &humanReadableBookId) {
void introduceTaskbar(string& content, const string& humanReadableBookId)
{
if (!noSearchBarFlag) {
content = appendToFirstOccurence(content, "<head>",
replaceRegex(RESOURCE::include_html_part,
humanReadableBookId, "__CONTENT__"));
content = appendToFirstOccurence(content,
content = appendToFirstOccurence(
content,
"<head>",
replaceRegex(
RESOURCE::include_html_part, humanReadableBookId, "__CONTENT__"));
content = appendToFirstOccurence(
content,
"<head>",
"<style>" +
RESOURCE::taskbar_css +
(noLibraryButtonFlag ? " #kiwix_serve_taskbar_library_button { display: none }" : "") +
"</style>");
content = appendToFirstOccurence(content, "<body[^>]*>",
replaceRegex(RESOURCE::taskbar_html_part,
humanReadableBookId, "__CONTENT__"));
"<style>" + RESOURCE::taskbar_css
+ (noLibraryButtonFlag
? " #kiwix_serve_taskbar_library_button { display: none }"
: "")
+ "</style>");
content = appendToFirstOccurence(
content,
"<body[^>]*>",
replaceRegex(
RESOURCE::taskbar_html_part, humanReadableBookId, "__CONTENT__"));
}
}
/* Should display debug information? */
bool isVerbose() {
bool isVerbose()
{
bool value;
pthread_mutex_lock(&verboseFlagLock);
value = verboseFlag;
@ -146,16 +156,14 @@ bool isVerbose() {
return value;
}
static
bool compress_content(string &content,
const string &mimeType)
static bool compress_content(string& content, const string& mimeType)
{
static std::vector<Bytef> compr_buffer;
/* Should be deflate */
bool deflated = mimeType.find("text/") != string::npos ||
mimeType.find("application/javascript") != string::npos ||
mimeType.find("application/json") != string::npos;
bool deflated = mimeType.find("text/") != string::npos
|| mimeType.find("application/javascript") != string::npos
|| mimeType.find("application/json") != string::npos;
if (!deflated)
return false;
@ -173,7 +181,10 @@ bool compress_content(string &content,
pthread_mutex_lock(&compressorLock);
compr_buffer.reserve(bufferBound);
uLongf comprLen = compr_buffer.capacity();
int err = compress(&compr_buffer[0], &comprLen, (const Bytef*)(content.data()), contentLength);
int err = compress(&compr_buffer[0],
&comprLen,
(const Bytef*)(content.data()),
contentLength);
if (err == Z_OK && comprLen > 2 && comprLen < (contentLength + 2)) {
/* /!\ Internet Explorer has a bug with deflate compression.
@ -190,9 +201,7 @@ bool compress_content(string &content,
return deflated;
}
static
struct MHD_Response* build_response(const void* data,
static struct MHD_Response* build_response(const void* data,
unsigned int length,
const std::string& httpRedirection,
const std::string& mimeType,
@ -200,40 +209,46 @@ struct MHD_Response* build_response(const void* data,
bool cacheEnabled)
{
/* Create the response */
struct MHD_Response * response = MHD_create_response_from_data(length,
const_cast<void*>(data),
MHD_NO,
MHD_YES);
struct MHD_Response* response = MHD_create_response_from_data(
length, const_cast<void*>(data), MHD_NO, MHD_YES);
/* Make a redirection if necessary otherwise send the content */
if (!httpRedirection.empty()) {
MHD_add_response_header(response, MHD_HTTP_HEADER_LOCATION, httpRedirection.c_str());
MHD_add_response_header(
response, MHD_HTTP_HEADER_LOCATION, httpRedirection.c_str());
} else {
/* Add if necessary the content-encoding */
if (deflated) {
MHD_add_response_header(response, MHD_HTTP_HEADER_VARY, "Accept-Encoding");
MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate");
MHD_add_response_header(
response, MHD_HTTP_HEADER_VARY, "Accept-Encoding");
MHD_add_response_header(
response, MHD_HTTP_HEADER_CONTENT_ENCODING, "deflate");
}
/* Tell the client that byte ranges are accepted */
MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes");
/* Specify the mime type */
MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_TYPE, mimeType.c_str());
MHD_add_response_header(
response, MHD_HTTP_HEADER_CONTENT_TYPE, mimeType.c_str());
}
/* Force to close the connection - cf. 100% CPU usage with v. 4.4 (in Lucid) */
/* Force to close the connection - cf. 100% CPU usage with v. 4.4 (in Lucid)
*/
// MHD_add_response_header(response, MHD_HTTP_HEADER_CONNECTION, "close");
/* Allow cross-domain requests */
//MHD_add_response_header(response, MHD_HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, "*");
// MHD_add_response_header(response,
// MHD_HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, "*");
MHD_add_response_header(response, "Access-Control-Allow-Origin", "*");
if (cacheEnabled) { /* Force cache */
MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL, "max-age=2723040, public");
MHD_add_response_header(
response, MHD_HTTP_HEADER_CACHE_CONTROL, "max-age=2723040, public");
} else { /* Prevent cache (for random page) */
MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL, "no-cache, no-store, must-revalidate");
MHD_add_response_header(response,
MHD_HTTP_HEADER_CACHE_CONTROL,
"no-cache, no-store, must-revalidate");
}
return response;
}
@ -247,8 +262,7 @@ ssize_t callback_reader_from_blob(void *cls,
pthread_mutex_lock(&readerLock);
size_t max_size_to_set = min<size_t>(max, blob->size() - pos);
if (max_size_to_set <= 0)
{
if (max_size_to_set <= 0) {
pthread_mutex_unlock(&readerLock);
return MHD_CONTENT_READER_END_WITH_ERROR;
}
@ -266,13 +280,13 @@ void callback_free_blob(void *cls)
pthread_mutex_unlock(&readerLock);
}
static
struct MHD_Response* build_callback_response_from_blob(zim::Blob& blob,
const std::string& mimeType)
static struct MHD_Response* build_callback_response_from_blob(
zim::Blob& blob, const std::string& mimeType)
{
pthread_mutex_lock(&readerLock);
zim::Blob* p_blob = new zim::Blob(blob);
struct MHD_Response * response = MHD_create_response_from_callback(blob.size(),
struct MHD_Response* response
= MHD_create_response_from_callback(blob.size(),
16384,
callback_reader_from_blob,
p_blob,
@ -282,19 +296,22 @@ struct MHD_Response* build_callback_response_from_blob(zim::Blob& blob,
MHD_add_response_header(response, MHD_HTTP_HEADER_ACCEPT_RANGES, "bytes");
/* Specify the mime type */
MHD_add_response_header(response, MHD_HTTP_HEADER_CONTENT_TYPE, mimeType.c_str());
MHD_add_response_header(
response, MHD_HTTP_HEADER_CONTENT_TYPE, mimeType.c_str());
/* Allow cross-domain requests */
//MHD_add_response_header(response, MHD_HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, "*");
// MHD_add_response_header(response,
// MHD_HTTP_HEADER_ACCESS_CONTROL_ALLOW_ORIGIN, "*");
MHD_add_response_header(response, "Access-Control-Allow-Origin", "*");
MHD_add_response_header(response, MHD_HTTP_HEADER_CACHE_CONTROL, "max-age=2723040, public");
MHD_add_response_header(
response, MHD_HTTP_HEADER_CACHE_CONTROL, "max-age=2723040, public");
return response;
}
static
struct MHD_Response* handle_suggest(struct MHD_Connection * connection,
static struct MHD_Response* handle_suggest(
struct MHD_Connection* connection,
int& httpResponseCode,
kiwix::Reader* reader,
kiwix::Searcher* searcher,
@ -309,7 +326,8 @@ struct MHD_Response* handle_suggest(struct MHD_Connection * connection,
std::string suggestion;
/* Get the suggestion pattern from the HTTP request */
const char* cTerm = MHD_lookup_connection_value(connection, MHD_GET_ARGUMENT_KIND, "term");
const char* cTerm
= MHD_lookup_connection_value(connection, MHD_GET_ARGUMENT_KIND, "term");
std::string term = cTerm == NULL ? "" : cTerm;
if (isVerbose()) {
std::cout << "Searching suggestions for: \"" << term << "\"" << endl;
@ -321,24 +339,27 @@ struct MHD_Response* handle_suggest(struct MHD_Connection * connection,
while (reader->getNextSuggestion(suggestion)) {
kiwix::stringReplacement(suggestion, "\"", "\\\"");
content += (content == "[" ? "" : ",");
content += "{\"value\":\"" + suggestion + "\",\"label\":\"" + suggestion + "\"}";
content += "{\"value\":\"" + suggestion + "\",\"label\":\"" + suggestion
+ "\"}";
suggestionCount++;
}
/* Propose the fulltext search if possible */
if (searcher != NULL) {
content += (suggestionCount == 0 ? "" : ",");
content += "{\"value\":\"" + std::string(term) + " \", \"label\":\"containing '" + std::string(term) + "'...\"}";
content += "{\"value\":\"" + std::string(term)
+ " \", \"label\":\"containing '" + std::string(term)
+ "'...\"}";
}
content += "]";
mimeType = "application/json; charset=utf-8";
bool deflated = acceptEncodingDeflate && compress_content(content, mimeType);
return build_response(content.data(), content.size(), "", mimeType, deflated, true);
return build_response(
content.data(), content.size(), "", mimeType, deflated, true);
}
static
struct MHD_Response* handle_skin(struct MHD_Connection * connection,
static struct MHD_Response* handle_skin(struct MHD_Connection* connection,
int& httpResponseCode,
kiwix::Reader* reader,
kiwix::Searcher* searcher,
@ -349,11 +370,12 @@ struct MHD_Response* handle_skin(struct MHD_Connection * connection,
std::string content = getResource(urlStr.substr(6));
std::string mimeType = getMimeTypeForFile(urlStr);
bool deflated = acceptEncodingDeflate && compress_content(content, mimeType);
return build_response(content.data(), content.size(), "", mimeType, deflated, true);
return build_response(
content.data(), content.size(), "", mimeType, deflated, true);
}
static
struct MHD_Response* handle_search(struct MHD_Connection * connection,
static struct MHD_Response* handle_search(
struct MHD_Connection* connection,
int& httpResponseCode,
kiwix::Reader* reader,
kiwix::Searcher* searcher,
@ -366,8 +388,10 @@ struct MHD_Response* handle_search(struct MHD_Connection * connection,
std::string httpRedirection;
/* Retrieve the pattern to search */
const char* pattern = MHD_lookup_connection_value(connection, MHD_GET_ARGUMENT_KIND, "pattern");
std::string patternString = kiwix::urlDecode(pattern == NULL ? "" : string(pattern));
const char* pattern = MHD_lookup_connection_value(
connection, MHD_GET_ARGUMENT_KIND, "pattern");
std::string patternString
= kiwix::urlDecode(pattern == NULL ? "" : string(pattern));
std::string patternCorrespondingUrl;
/* Try first to load directly the article */
@ -384,7 +408,8 @@ struct MHD_Response* handle_search(struct MHD_Connection * connection,
/* If article found then redirect directly to it */
if (!patternCorrespondingUrl.empty()) {
httpRedirection = "/" + humanReadableBookId + "/" + patternCorrespondingUrl;
httpRedirection
= "/" + humanReadableBookId + "/" + patternCorrespondingUrl;
httpResponseCode = MHD_HTTP_FOUND;
return build_response("", 0, httpRedirection, "", false, true);
}
@ -392,8 +417,10 @@ struct MHD_Response* handle_search(struct MHD_Connection * connection,
/* Make the search */
if (patternCorrespondingUrl.empty() && searcher != NULL) {
const char* start = MHD_lookup_connection_value(connection, MHD_GET_ARGUMENT_KIND, "start");
const char* end = MHD_lookup_connection_value(connection, MHD_GET_ARGUMENT_KIND, "end");
const char* start = MHD_lookup_connection_value(
connection, MHD_GET_ARGUMENT_KIND, "start");
const char* end
= MHD_lookup_connection_value(connection, MHD_GET_ARGUMENT_KIND, "end");
unsigned int startNumber = start != NULL ? atoi(start) : 0;
unsigned int endNumber = end != NULL ? atoi(end) : 25;
@ -418,11 +445,16 @@ struct MHD_Response* handle_search(struct MHD_Connection * connection,
introduceTaskbar(content, humanReadableBookId);
bool deflated = acceptEncodingDeflate && compress_content(content, mimeType);
return build_response(content.data(), content.size(), httpRedirection, mimeType, deflated, true);
return build_response(content.data(),
content.size(),
httpRedirection,
mimeType,
deflated,
true);
}
static
struct MHD_Response* handle_random(struct MHD_Connection * connection,
static struct MHD_Response* handle_random(
struct MHD_Connection* connection,
int& httpResponseCode,
kiwix::Reader* reader,
kiwix::Searcher* searcher,
@ -436,13 +468,14 @@ struct MHD_Response* handle_random(struct MHD_Connection * connection,
pthread_mutex_lock(&readerLock);
std::string randomUrl = reader->getRandomPageUrl();
pthread_mutex_unlock(&readerLock);
httpRedirection = "/" + humanReadableBookId + "/" + kiwix::urlEncode(randomUrl);
httpRedirection
= "/" + humanReadableBookId + "/" + kiwix::urlEncode(randomUrl);
}
return build_response("", 0, httpRedirection, "", false, false);
}
static
struct MHD_Response* handle_content(struct MHD_Connection * connection,
static struct MHD_Response* handle_content(
struct MHD_Connection* connection,
int& httpResponseCode,
kiwix::Reader* reader,
kiwix::Searcher* searcher,
@ -481,12 +514,19 @@ struct MHD_Response* handle_content(struct MHD_Connection * connection,
if (isVerbose())
cout << "Failed to find " << urlStr << endl;
content = "<!DOCTYPE html>\n<html><head><meta content=\"text/html;charset=UTF-8\" http-equiv=\"content-type\" /><title>Content not found</title></head><body><h1>Not Found</h1><p>The requested URL \"" + urlStr + "\" was not found on this server.</p></body></html>";
content
= "<!DOCTYPE html>\n<html><head><meta "
"content=\"text/html;charset=UTF-8\" http-equiv=\"content-type\" "
"/><title>Content not found</title></head><body><h1>Not "
"Found</h1><p>The requested URL \""
+ urlStr + "\" was not found on this server.</p></body></html>";
mimeType = "text/html";
httpResponseCode = MHD_HTTP_NOT_FOUND;
introduceTaskbar(content, humanReadableBookId);
bool deflated = acceptEncodingDeflate && compress_content(content, mimeType);
return build_response(content.data(), content.size(), "", mimeType, deflated, false);
bool deflated
= acceptEncodingDeflate && compress_content(content, mimeType);
return build_response(
content.data(), content.size(), "", mimeType, deflated, false);
}
try {
@ -506,41 +546,46 @@ struct MHD_Response* handle_content(struct MHD_Connection * connection,
zim::Blob raw_content = article.getData();
pthread_mutex_unlock(&readerLock);
if (mimeType.find("text/") != string::npos ||
mimeType.find("application/javascript") != string::npos ||
mimeType.find("application/json") != string::npos)
{
if (mimeType.find("text/") != string::npos
|| mimeType.find("application/javascript") != string::npos
|| mimeType.find("application/json") != string::npos) {
pthread_mutex_lock(&readerLock);
content = string(raw_content.data(), raw_content.size());
pthread_mutex_unlock(&readerLock);
/* Special rewrite URL in case of ZIM file use intern *asbolute* url like /A/Kiwix */
/* Special rewrite URL in case of ZIM file use intern *asbolute* url like
* /A/Kiwix */
if (mimeType.find("text/html") != string::npos) {
baseUrl = "/" + std::string(1, article.getNamespace()) + "/" + article.getUrl();
content = replaceRegex(content, "$1$2" + humanReadableBookId + "/$3/",
baseUrl = "/" + std::string(1, article.getNamespace()) + "/"
+ article.getUrl();
content = replaceRegex(content,
"$1$2" + humanReadableBookId + "/$3/",
"(href|src)(=[\"|\']{0,1}/)([A-Z|\\-])/");
content = replaceRegex(content, "$1$2" + humanReadableBookId + "/$3/",
"(@import[ ]+)([\"|\']{0,1}/)([A-Z|\\-])/");
content = replaceRegex(content,
"$1$2" + humanReadableBookId + "/$3/",
"(@import[ ]+)([\"|\']{0,1}/)([A-Z|\\-])/");
content = replaceRegex(
content,
"<head><base href=\"/" + humanReadableBookId + baseUrl + "\" />",
"<head>");
introduceTaskbar(content, humanReadableBookId);
} else if (mimeType.find("text/css") != string::npos) {
content = replaceRegex(content, "$1$2" + humanReadableBookId + "/$3/",
content = replaceRegex(content,
"$1$2" + humanReadableBookId + "/$3/",
"(url|URL)(\\([\"|\']{0,1}/)([A-Z|\\-])/");
}
bool deflated = acceptEncodingDeflate && compress_content(content, mimeType);
return build_response(content.data(), content.size(), "", mimeType, deflated, true);
}
else
{
bool deflated
= acceptEncodingDeflate && compress_content(content, mimeType);
return build_response(
content.data(), content.size(), "", mimeType, deflated, true);
} else {
return build_callback_response_from_blob(raw_content, mimeType);
}
}
static
struct MHD_Response* handle_default(struct MHD_Connection * connection,
static struct MHD_Response* handle_default(
struct MHD_Connection* connection,
int& httpResponseCode,
kiwix::Reader* reader,
kiwix::Searcher* searcher,
@ -555,7 +600,8 @@ struct MHD_Response* handle_default(struct MHD_Connection * connection,
std::string mimeType = "text/html; charset=utf-8";
bool deflated = acceptEncodingDeflate && compress_content(content, mimeType);
return build_response(content.data(), content.size(), "", mimeType, deflated, true);
return build_response(
content.data(), content.size(), "", mimeType, deflated, true);
}
static int accessHandlerCallback(void* cls,
@ -571,7 +617,8 @@ static int accessHandlerCallback(void *cls,
if (0 != strcmp(method, "GET") && 0 != strcmp(method, "POST"))
return MHD_NO;
/* The first time only the headers are valid, do not respond in the first round... */
/* The first time only the headers are valid, do not respond in the first
* round... */
static int dummy;
if (&dummy != *ptr) {
*ptr = &dummy;
@ -587,11 +634,15 @@ static int accessHandlerCallback(void *cls,
}
/* Check if the response can be compressed */
const char* acceptEncodingHeaderValue = MHD_lookup_connection_value(connection, MHD_HEADER_KIND, MHD_HTTP_HEADER_ACCEPT_ENCODING);
const bool acceptEncodingDeflate = acceptEncodingHeaderValue && string(acceptEncodingHeaderValue).find("deflate") != string::npos;
const char* acceptEncodingHeaderValue = MHD_lookup_connection_value(
connection, MHD_HEADER_KIND, MHD_HTTP_HEADER_ACCEPT_ENCODING);
const bool acceptEncodingDeflate
= acceptEncodingHeaderValue
&& string(acceptEncodingHeaderValue).find("deflate") != string::npos;
/* Check if range is requested. [TODO] Handle this somehow */
const char* acceptRangeHeaderValue = MHD_lookup_connection_value(connection, MHD_HEADER_KIND, MHD_HTTP_HEADER_RANGE);
const char* acceptRangeHeaderValue = MHD_lookup_connection_value(
connection, MHD_HEADER_KIND, MHD_HTTP_HEADER_RANGE);
const bool acceptRange = acceptRangeHeaderValue != NULL;
/* Prepare the variables */
@ -602,24 +653,32 @@ static int accessHandlerCallback(void *cls,
/* Get searcher and reader */
std::string humanReadableBookId = "";
if (!(urlStr.size() > 5 && urlStr.substr(0, 6) == "/skin/")) {
if (!strcmp(url, "/search") || !strcmp(url, "/suggest") || !strcmp(url, "/random")) {
const char* tmpGetValue = MHD_lookup_connection_value(connection, MHD_GET_ARGUMENT_KIND, "content");
if (!strcmp(url, "/search") || !strcmp(url, "/suggest")
|| !strcmp(url, "/random")) {
const char* tmpGetValue = MHD_lookup_connection_value(
connection, MHD_GET_ARGUMENT_KIND, "content");
humanReadableBookId = (tmpGetValue != NULL ? string(tmpGetValue) : "");
} else {
humanReadableBookId = urlStr.substr(1, urlStr.find("/", 1) != string::npos ?
urlStr.find("/", 1) - 1 : urlStr.size() - 2);
humanReadableBookId = urlStr.substr(1,
urlStr.find("/", 1) != string::npos
? urlStr.find("/", 1) - 1
: urlStr.size() - 2);
if (!humanReadableBookId.empty()) {
urlStr = urlStr.substr(urlStr.find("/", 1) != string::npos ?
urlStr.find("/", 1) : humanReadableBookId.size());
urlStr = urlStr.substr(urlStr.find("/", 1) != string::npos
? urlStr.find("/", 1)
: humanReadableBookId.size());
}
}
}
pthread_mutex_lock(&mapLock);
kiwix::Searcher *searcher = searchers.find(humanReadableBookId) != searchers.end() ?
searchers.find(humanReadableBookId)->second : NULL;
kiwix::Reader *reader = readers.find(humanReadableBookId) != readers.end() ?
readers.find(humanReadableBookId)->second : NULL;
kiwix::Searcher* searcher
= searchers.find(humanReadableBookId) != searchers.end()
? searchers.find(humanReadableBookId)->second
: NULL;
kiwix::Reader* reader = readers.find(humanReadableBookId) != readers.end()
? readers.find(humanReadableBookId)->second
: NULL;
if (reader == NULL) {
humanReadableBookId = "";
}
@ -692,15 +751,14 @@ static int accessHandlerCallback(void *cls,
}
/* Queue the response */
int ret = MHD_queue_response(connection,
httpResponseCode,
response);
int ret = MHD_queue_response(connection, httpResponseCode, response);
MHD_destroy_response(response);
return ret;
}
int main(int argc, char **argv) {
int main(int argc, char** argv)
{
struct MHD_Daemon* daemon;
vector<string> zimPathes;
string libraryPath;
@ -714,8 +772,8 @@ int main(int argc, char **argv) {
unsigned int PPID = 0;
kiwix::Manager libraryManager;
static struct option long_options[] = {
{"daemon", no_argument, 0, 'd'},
static struct option long_options[]
= {{"daemon", no_argument, 0, 'd'},
{"verbose", no_argument, 0, 'v'},
{"library", no_argument, 0, 'l'},
{"nolibrarybutton", no_argument, 0, 'm'},
@ -724,16 +782,15 @@ int main(int argc, char **argv) {
{"attachToProcess", required_argument, 0, 'a'},
{"port", required_argument, 0, 'p'},
{"interface", required_argument, 0, 'f'},
{0, 0, 0, 0}
};
{0, 0, 0, 0}};
/* Argument parsing */
while (true) {
int option_index = 0;
int c = getopt_long(argc, argv, "mndvli:a:p:f:", long_options, &option_index);
int c
= getopt_long(argc, argv, "mndvli:a:p:f:", long_options, &option_index);
if (c != -1) {
switch (c) {
case 'd':
daemonFlag = true;
@ -766,8 +823,7 @@ int main(int argc, char **argv) {
}
} else {
if (optind <= argc) {
if (libraryFlag)
{
if (libraryFlag) {
libraryPath = argv[optind++];
} else {
while (optind < argc)
@ -780,9 +836,17 @@ int main(int argc, char **argv) {
/* Print usage)) if necessary */
if (zimPathes.empty() && libraryPath.empty()) {
cerr << "Usage: kiwix-serve [--index=INDEX_PATH] [--port=PORT] [--verbose] [--nosearchbar] [--nolibrarybutton] [--daemon] [--attachToProcess=PID] [--interface=IF_NAME] ZIM_PATH+" << endl;
cerr << " kiwix-serve --library [--port=PORT] [--verbose] [--daemon] [--nosearchbar] [--nolibrarybutton] [--attachToProcess=PID] [--interface=IF_NAME] LIBRARY_PATH" << endl;
cerr << "\n If you set more than one ZIM_PATH, you cannot set a INDEX_PATH." << endl;
cerr << "Usage: kiwix-serve [--index=INDEX_PATH] [--port=PORT] [--verbose] "
"[--nosearchbar] [--nolibrarybutton] [--daemon] "
"[--attachToProcess=PID] [--interface=IF_NAME] ZIM_PATH+"
<< endl;
cerr << " kiwix-serve --library [--port=PORT] [--verbose] [--daemon] "
"[--nosearchbar] [--nolibrarybutton] [--attachToProcess=PID] "
"[--interface=IF_NAME] LIBRARY_PATH"
<< endl;
cerr << "\n If you set more than one ZIM_PATH, you cannot set a "
"INDEX_PATH."
<< endl;
exit(1);
}
@ -801,14 +865,20 @@ int main(int argc, char **argv) {
bool retVal = false;
try {
string libraryPath = isRelativePath(*itr) ? computeAbsolutePath(removeLastPathElement(getExecutablePath(), true, false), *itr) : *itr;
string libraryPath
= isRelativePath(*itr)
? computeAbsolutePath(removeLastPathElement(
getExecutablePath(), true, false),
*itr)
: *itr;
retVal = libraryManager.readFile(libraryPath, true);
} catch (...) {
retVal = false;
}
if (!retVal) {
cerr << "Unable to open the XML library file '" << *itr << "'." << endl;
cerr << "Unable to open the XML library file '" << *itr << "'."
<< endl;
exit(1);
}
}
@ -816,13 +886,15 @@ int main(int argc, char **argv) {
/* Check if the library is not empty (or only remote books)*/
if (libraryManager.getBookCount(true, false) == 0) {
cerr << "The XML library file '" << libraryPath << "' is empty (or has only remote books)." << endl;
cerr << "The XML library file '" << libraryPath
<< "' is empty (or has only remote books)." << endl;
}
} else {
std::vector<std::string>::iterator it;
for (it = zimPathes.begin(); it != zimPathes.end(); it++) {
if (!libraryManager.addBookFromPath(*it, *it, "", false)) {
cerr << "Unable to add the ZIM file '" << *it << "' to the internal library." << endl;
cerr << "Unable to add the ZIM file '" << *it
<< "' to the internal library." << endl;
exit(1);
}
}
@ -871,7 +943,8 @@ int main(int argc, char **argv) {
searcher->setContentHumanReadableId(humanReadableId);
searchers[humanReadableId] = searcher;
} catch (...) {
cerr << "Unable to open the search index '" << indexPath << "'." << endl;
cerr << "Unable to open the search index '" << indexPath << "'."
<< endl;
}
}
}
@ -879,12 +952,15 @@ int main(int argc, char **argv) {
}
/* Compute the Welcome HTML */
string welcomeBooksHtml = ""
string welcomeBooksHtml
= ""
"<div class='book__list'>";
for (itr = booksIds.begin(); itr != booksIds.end(); ++itr) {
libraryManager.getBookById(*itr, currentBook);
if (!currentBook.path.empty() && readers.find(currentBook.getHumanReadableIdFromPath()) != readers.end()) {
if (!currentBook.path.empty()
&& readers.find(currentBook.getHumanReadableIdFromPath())
!= readers.end()) {
welcomeBooksHtml += ""
"<a href='/" + currentBook.getHumanReadableIdFromPath() + "/'>"
"<div class='book'>"
@ -902,7 +978,8 @@ int main(int argc, char **argv) {
welcomeBooksHtml += ""
"</div>";
welcomeHTML = replaceRegex(RESOURCE::home_html_tmpl, welcomeBooksHtml, "__BOOKS__");
welcomeHTML
= replaceRegex(RESOURCE::home_html_tmpl, welcomeBooksHtml, "__BOOKS__");
#ifndef _WIN32
/* Fork if necessary */
@ -956,7 +1033,6 @@ int main(int argc, char **argv) {
/* Start the HTTP daemon */
void* page = NULL;
if (interface.length() > 0) {
#ifndef _WIN32
/* TBD IPv6 support */
@ -966,7 +1042,8 @@ int main(int argc, char **argv) {
/* Search all available interfaces */
if (getifaddrs(&ifaddr) == -1) {
cerr << "Getifaddrs() failed while searching for '" << interface << "'" << endl;
cerr << "Getifaddrs() failed while searching for '" << interface << "'"
<< endl;
exit(1);
}
@ -975,7 +1052,6 @@ int main(int argc, char **argv) {
/* Try to find interfaces in the list of available interfaces */
for (ifa = ifaddr, n = 0; ifa != NULL; ifa = ifa->ifa_next, n++) {
/* Ignore if no IP attributed to the interface */
if (ifa->ifa_addr == NULL)
continue;
@ -986,7 +1062,8 @@ int main(int argc, char **argv) {
if (strcasecmp(ifa->ifa_name, interface.c_str()) == 0) {
sockAddr.sin_family = family;
sockAddr.sin_port = htons(serverPort);
sockAddr.sin_addr.s_addr = ((struct sockaddr_in*) ifa->ifa_addr)->sin_addr.s_addr;
sockAddr.sin_addr.s_addr
= ((struct sockaddr_in*)ifa->ifa_addr)->sin_addr.s_addr;
break;
}
}
@ -1026,7 +1103,10 @@ int main(int argc, char **argv) {
}
if (daemon == NULL) {
cerr << "Unable to instanciate the HTTP daemon. The port " << serverPort << " is maybe already occupied or need more permissions to be open. Please try as root or with a port number higher or equal to 1024." << endl;
cerr << "Unable to instanciate the HTTP daemon. The port " << serverPort
<< " is maybe already occupied or need more permissions to be open. "
"Please try as root or with a port number higher or equal to 1024."
<< endl;
exit(1);
}

Write Preview
Loading…
Cancel
Save