diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..aa1c1e4 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,61 @@ +name: Docker + +on: + workflow_dispatch: + inputs: + version: + description: Specific version to build (overrides on-master and tag-pattern) + required: false + default: '' + +jobs: + build-and-push-kiwix-tools: + name: Deploy kiwix-tools Docker Image + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: build and publish kiwix-tools + uses: openzim/docker-publish-action@v5 + with: + image-name: kiwix/kiwix-tools + credentials: | + DOCKERIO_USERNAME=${{ secrets.DOCKERHUB_USERNAME }} + DOCKERIO_TOKEN=${{ secrets.DOCKERHUB_PASSWORD }} + GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }} + GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }} + context: docker + latest-on-tag: true + build-args: + VERSION={tag} + platforms: | + linux/amd64 + linux/arm/v7 + linux/arm64 + restrict-to: kiwix/kiwix-tools + manual-tag: ${{ github.event.inputs.version }} + + build-and-push-kiwix-serve: + name: Deploy kiwix-serve Docker Image + runs-on: ubuntu-20.04 + needs: build-and-push-kiwix-tools + steps: + - uses: actions/checkout@v2 + - name: build and publish kiwix-serve + uses: openzim/docker-publish-action@v5 + with: + image-name: kiwix/kiwix-serve + credentials: | + DOCKERIO_USERNAME=${{ secrets.DOCKERHUB_USERNAME }} + DOCKERIO_TOKEN=${{ secrets.DOCKERHUB_PASSWORD }} + GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }} + GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }} + context: docker/server + latest-on-tag: true + build-args: + VERSION={tag} + platforms: | + linux/amd64 + linux/arm/v7 + linux/arm64 + restrict-to: kiwix/kiwix-tools + manual-tag: ${{ github.event.inputs.version }} diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml new file mode 100644 index 0000000..b9bf3b5 --- /dev/null +++ b/.github/workflows/package.yml @@ -0,0 +1,79 @@ +name: Packages +on: [push, pull_request] + +jobs: + build-deb: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + distro: [ubuntu-hirsute, ubuntu-groovy, ubuntu-focal] + steps: + - uses: actions/checkout@v2 + + # Determine which PPA we should upload to + - name: PPA + id: ppa + run: | + if [[ $REF == refs/tags* ]] + then + echo "::set-output name=ppa::kiwixteam/release" + else + echo "::set-output name=ppa::kiwixteam/dev" + fi + env: + REF: ${{ github.ref }} + + - uses: legoktm/gh-action-auto-dch@master + with: + fullname: Kiwix builder + email: release+launchpad@kiwix.org + distro: ${{ matrix.distro }} + + - uses: legoktm/gh-action-build-deb@ubuntu-hirsute + if: matrix.distro == 'ubuntu-hirsute' + name: Build package for ubuntu-hirsute + id: build-ubuntu-hirsute + with: + args: --no-sign + ppa: ${{ steps.ppa.outputs.ppa }} + + - uses: legoktm/gh-action-build-deb@ubuntu-groovy + if: matrix.distro == 'ubuntu-groovy' + name: Build package for ubuntu-groovy + id: build-ubuntu-groovy + with: + args: --no-sign + ppa: ${{ steps.ppa.outputs.ppa }} + + - uses: legoktm/gh-action-build-deb@ubuntu-focal + if: matrix.distro == 'ubuntu-focal' + name: Build package for ubuntu-focal + id: build-ubuntu-focal + with: + args: --no-sign + ppa: ${{ steps.ppa.outputs.ppa }} + + - uses: actions/upload-artifact@v2 + with: + name: Packages for ${{ matrix.distro }} + path: output + + - uses: legoktm/gh-action-dput@master + name: Upload dev package + # Only upload on pushes to master + if: github.event_name == 'push' && github.event.ref == 'refs/heads/master' && startswith(matrix.distro, 'ubuntu-') + with: + gpg_key: ${{ secrets.LAUNCHPAD_GPG }} + repository: ppa:kiwixteam/dev + packages: output/*_source.changes + + - uses: legoktm/gh-action-dput@master + name: Upload release package + # Only upload on pushes to master or tag + if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') && startswith(matrix.distro, 'ubuntu-') + with: + gpg_key: ${{ secrets.LAUNCHPAD_GPG }} + repository: ppa:kiwixteam/release + packages: output/*_source.changes + diff --git a/README.md b/README.md index 97c2bd2..854802b 100644 --- a/README.md +++ b/README.md @@ -8,22 +8,14 @@ command line tools: * kiwix-search: Fulltext search in ZIM files * kiwix-serve: HTTP daemon serving ZIM files +[![latest release](https://img.shields.io/github/v/tag/kiwix/kiwix-tools?label=latest%20release&sort=semver)](https://download.kiwix.org/release/kiwix-tools/) +[![Docker](https://img.shields.io/docker/v/kiwix/kiwix-serve?arch=amd64&label=Docker&sort=date)](https://hub.docker.com/r/kiwix/kiwix-serve) +[![Sandstorm](https://img.shields.io/badge/Sandstorm-kiwix-blue)](https://apps.sandstorm.io/app/5uh349d0kky2zp5whrh2znahn27gwha876xze3864n0fu9e5220h) [![Build Status](https://github.com/kiwix/kiwix-tools/workflows/CI/badge.svg?query=branch%3Amaster)](https://github.com/kiwix/kiwix-tools/actions?query=branch%3Amaster) [![CodeFactor](https://www.codefactor.io/repository/github/kiwix/kiwix-tools/badge)](https://www.codefactor.io/repository/github/kiwix/kiwix-tools) [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0) -Distribution ------------- - -#### Official - -[![latest release](https://img.shields.io/github/v/tag/kiwix/kiwix-tools?label=latest%20release&sort=semver)](https://download.kiwix.org/release/kiwix-tools/) -[![Docker](https://img.shields.io/docker/cloud/build/kiwix/kiwix-serve)](https://hub.docker.com/r/kiwix/kiwix-serve) - -#### Contribution - -[![AUR](https://img.shields.io/aur/version/kiwix-tools)](https://aur.archlinux.org/packages/kiwix-tools/) -[![Sandstorm](https://img.shields.io/badge/Sandstorm-kiwix-blue)](https://apps.sandstorm.io/app/5uh349d0kky2zp5whrh2znahn27gwha876xze3864n0fu9e5220h) +[![Packaging status](https://repology.org/badge/vertical-allrepos/kiwix-tools.svg)](https://repology.org/project/kiwix-tools/versions) Disclaimer ---------- diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..c8c2fa0 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +kiwix-tools (0.0.0) unstable; urgency=medium + + * Initial release + + -- Kunal Mehta Mon, 13 Jul 2020 17:21:11 -0700 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..c23bf55 --- /dev/null +++ b/debian/control @@ -0,0 +1,27 @@ +Source: kiwix-tools +Section: utils +Priority: optional +Maintainer: Kiwix team +Build-Depends: debhelper-compat (= 13), + libkiwix-dev (>= 9.3.0~), + libmicrohttpd-dev, + meson, + pkg-config, + zlib1g-dev +Standards-Version: 4.5.0 +Homepage: https://github.com/kiwix/kiwix-tools +Rules-Requires-Root: no + +Package: kiwix-tools +Architecture: any +Depends: ${misc:Depends}, ${shlibs:Depends} +Description: collection of Kiwix tools + kiwix-tools is a collection of various command-line tools used to help + users interact with and manage ZIM files. It includes: + * kiwix-serve is a standalone HTTP server for serving ZIM files + over the network. + * kiwix-manage allows one to manage the content of the Kiwix library (an + XML file listing available ZIM files). + * kiwix-read allows for reading ZIM files from the command-line. + * kiwix-search allows one to find articles in a ZIM file using fulltext + search patterns. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..ff46366 --- /dev/null +++ b/debian/copyright @@ -0,0 +1 @@ +See COPYING in the repository root. diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..098338c --- /dev/null +++ b/debian/rules @@ -0,0 +1,5 @@ +#!/usr/bin/make -f +export DEB_BUILD_MAINT_OPTIONS = hardening=+all + +%: + dh $@ diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000..89ae9db --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (native) diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..8f80b75 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,33 @@ +# declare build option ARCH if not using buildx +ARG ARCH= +# declare version to build image for +ARG VERSION= + +# alpine is a multi-arch image +FROM alpine:3 +LABEL org.opencontainers.image.source https://github.com/openzim/kiwix-tools + +# TARGETARCH is injected by buildx +ARG TARGETARCH +ARG VERSION +ARG ARCH + +# if we had no TARGETARCH (not buildx), use --build-arg ARCH. default to amd64 +RUN echo "amd64" > /etc/docker_arch +RUN if [ ! -z "$ARCH" ] ; then echo "$ARCH" > /etc/docker_arch ; fi +RUN if [ ! -z "$TARGETARCH" ] ; then echo "$TARGETARCH" > /etc/docker_arch ; fi + +# decide which kiwix arch to download later (`armhf` for all arm* and x86_64 otherwise) +RUN if [ $(cut -c 1-3 /etc/docker_arch) = "arm" ] ; then echo "armhf" > /etc/kiwix_arch ; else echo "x86_64" > /etc/kiwix_arch ; fi + +# Install kiwix-tools +RUN url="http://mirror.download.kiwix.org/release/kiwix-tools/kiwix-tools_linux-$(cat /etc/kiwix_arch)-$VERSION.tar.gz" && \ + echo "URL: $url" && \ + wget -O - $url | tar -xz && \ + mv kiwix-tools*/kiwix-* /usr/local/bin && \ + rm -r kiwix-tools* + +# expose kiwix-serve default port +EXPOSE 80 + +CMD ["/bin/sh", "-c", "echo 'Welcome to kiwix-tools! The following binaries are available:' && ls /usr/local/bin/"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..4daccbe --- /dev/null +++ b/docker/README.md @@ -0,0 +1,32 @@ +Kiwix-tools Docker image +=== + +- Available on [docker.io](https://hub.docker.com/r/kiwix/kiwix-tools) and [ghcr.io](https://ghcr.io/kiwix/kiwix-tools). +- multi-arch (`linux/amd64`, `linux/arm64`, `linux/arm/v7`) +- based on official `kiwix-tools` binaries. + +## Usage + +``` sh +$ docker run -it kiwix/kiwix-tools:3.1.2 + +Welcome to kiwix-tools! The following binaries are available: +kiwix-manage kiwix-read kiwix-search kiwix-serve +``` + +`kiwix-tools` operates on zim files. You shall mount a volume to access the files. + +```sh +docker run -v $(pwd):/data -it kiwix/kiwix-tools kiwix-read --suggest="Mali" /data/wikipedia_fr_test.zim +``` + +## Building and reusing + +- `kiwix/kiwix-tools` is multi-arch and is ideally built using `buildx`. +- requires a `--build-arg VERSION=` with the kiwix-tools release. +- can be built using `docker build` in which case it expects an additionnal `--build-arg ARCH=arm` for arm. Otherwise defaults to `amd64`. + +**Notes:** + +- `wget` in `alpine:3` on `arm/v7` (__inside github action only__) crashes when downloading from HTTPs locations. Keep http-only in Dockerfile. +- Was also unhappy when using the mirrors so it's using `mirror.download` on purpose. \ No newline at end of file diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index f0b4017..48c63ac 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,14 +1,10 @@ -FROM alpine:latest -LABEL maintainer Emmanuel Engelhart +ARG VERSION=latest -# Install kiwix-serve -WORKDIR / -RUN apk add --no-cache curl bzip2 -RUN curl -kL https://download.kiwix.org/release/kiwix-tools/kiwix-tools_linux-x86_64.tar.gz | tar -xz && \ - mv kiwix-tools*/kiwix-serve /usr/local/bin && \ - rm -r kiwix-tools* +# kiwix-tools is multi-arch +FROM kiwix/kiwix-tools:$VERSION +LABEL org.opencontainers.image.source https://github.com/openzim/kiwix-tools -# Run kiwix-serve +# expose kiwix-serve default port and workdir EXPOSE 80 VOLUME /data WORKDIR /data diff --git a/docker/server/README.md b/docker/server/README.md index 49980b8..9b5162f 100644 --- a/docker/server/README.md +++ b/docker/server/README.md @@ -1,15 +1,15 @@ Kiwix serve Docker image ======================== -With local ZIM file -------------------- +With local ZIM file(s) +---------------------- * Download a ZIM file from -* Given `wikipedia.zim` resides in `/tmp/zim/`, execute the following: +* Given `wikipedia.zim` and `wiktionary.zim` reside in `/tmp/zim/`, execute the following: ```bash -docker run -v /tmp/zim:/data -p 8080:80 kiwix/kiwix-serve wikipedia.zim +docker run -v /tmp/zim:/data -p 8080:80 kiwix/kiwix-serve wikipedia.zim wiktionary.zim ``` With remote ZIM file @@ -19,6 +19,14 @@ With remote ZIM file docker run -e "DOWNLOAD=https://download.kiwix.org/zim/wikipedia_bm_all.zim" -p 8080:80 kiwix/kiwix-serve ``` +ARM +--- + +Build an image for an ARM based GNU/Linux: +```bash +docker build . -t kiwix/kiwix-serve:latest --build-arg ARCH="arm32v7/" +``` + Screenshots ----------- diff --git a/src/searcher/kiwix-search.cpp b/src/searcher/kiwix-search.cpp index b56d543..61f5806 100644 --- a/src/searcher/kiwix-search.cpp +++ b/src/searcher/kiwix-search.cpp @@ -104,7 +104,14 @@ int main(int argc, char** argv) if (reader) { searcher = new kiwix::Searcher(); - searcher->add_reader(reader); + bool contians_FTIndex=searcher->add_reader(reader); + if(!contians_FTIndex){ + std::cerr << "The Zim file does not contain a full-text index." << std::endl; + if(suggestionFlag){ + exit(0); + } + exit(1); + } } else { cerr << "Unable to search through zim '" << zimPath << "'." << endl; exit(1); diff --git a/src/server/kiwix-serve.cpp b/src/server/kiwix-serve.cpp index 9ac2083..ae742aa 100644 --- a/src/server/kiwix-serve.cpp +++ b/src/server/kiwix-serve.cpp @@ -96,7 +96,6 @@ int main(int argc, char** argv) bool blockExternalLinks = false; bool isVerboseFlag = false; bool trustlibrary = true; - string PPIDString; unsigned int PPID = 0; static struct option long_options[] @@ -154,7 +153,6 @@ int main(int argc, char** argv) serverPort = atoi(optarg); break; case 'a': - PPIDString = string(optarg); PPID = atoi(optarg); break; case 'i': @@ -289,7 +287,7 @@ int main(int argc, char** argv) int ret = sysctl(mib, MIBSIZE, &kp, &len, NULL, 0); if (ret != -1 && len > 0) { #else /* Linux & co */ - string procPath = "/proc/" + string(PPIDString); + string procPath = "/proc/" + std::to_string(PPID); if (access(procPath.c_str(), F_OK) != -1) { #endif } else {