Merge pull request #1 from freva/freva/devel

Initial release

Author: freva

.github/workflows/main.yml

@@ -0,0 +1,90 @@
+name: Main pipeline
+
+on:
+  push:
+    branches: [ "master" ]
+  pull_request:
+  release:
+    types: [ published ]
+
+permissions:
+  contents: write
+
+jobs:
+  frontend:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v3
+        with:
+          version: 10
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: 22
+          cache: 'pnpm'
+          cache-dependency-path: frontend/pnpm-lock.yaml
+
+      - name: Install Dependencies
+        run: pnpm install -C frontend --frozen-lockfile
+
+      - name: Typecheck
+        working-directory: frontend
+        run: pnpm typecheck
+
+      - name: Lint
+        working-directory: frontend
+        run: pnpm lint
+
+      - name: Test
+        working-directory: frontend
+        run: pnpm test
+
+      - name: Build Frontend
+        run: make ui
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+
+      - name: Test Backend
+        run: make test
+
+      - name: Build Go Binaries
+        run: |
+          # os/arch/extension
+          TARGETS=("linux/amd64/" "linux/arm64/" "darwin/amd64/" "darwin/arm64/" "windows/amd64/.exe")
+          
+          for target in "${TARGETS[@]}"; do
+            IFS="/" read -r OS ARCH EXT <<< "$target"
+          
+            echo "Building for $OS-$ARCH..."
+            PLATFORM_DIR="dist/${OS}_${ARCH}"
+            mkdir -p "$PLATFORM_DIR"
+          
+            for d in cmd/*/; do
+              binary_name=$(basename "$d")
+              GOOS=$OS GOARCH=$ARCH go build -o "${PLATFORM_DIR}/${binary_name}${EXT}" "./$d"
+            done
+          
+            VERSION=${{ github.event.release.tag_name || 'dev' }}
+            if [ "$OS" == "windows" ]; then
+              cd dist && zip -r "../codesearch-${VERSION}-${OS}-${ARCH}.zip" "${OS}_${ARCH}" && cd ..
+            else
+              tar -cvzf "codesearch-${VERSION}-${OS}-${ARCH}.tar.gz" -C dist "${OS}_${ARCH}"
+            fi
+          done
+
+      - name: Upload to Release
+        if: github.event_name == 'release'
+        uses: softprops/action-gh-release@v2
+        with:
+          files: |
+            *.tar.gz
+            *.zip
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -0,0 +1,6 @@
+.idea/
+
+/cmd/cserver/static/
+/db/
+/config
+/Makefile

AUTHORS

@@ -0,0 +1,15 @@
+all: ui go
+all-restart: all
+	systemctl --user restart codesearch-server.service
+
+go:
+	go install ./...
+
+test:
+	go test ./...
+
+ui:
+	cd frontend && pnpm install && pnpm build --emptyOutDir --outDir ../cmd/cserver/static
+
+update:
+	~/.go/bin/csupdater --config ./config

CONTRIBUTORS

@@ -0,0 +1,111 @@
+# Code search
+
+Extends [github.com/google/codesearch](https://github.com/google/codesearch) with a tool to sync git repositories
+to be indexed and a web interface. The sync tool and web interface inspired by [github.com/hakonhall/codesearch](https://github.com/hakonhall/codesearch).
+
+## Configuration
+
+Configuration is done via a config file. The format is as follows:
+
+- The file consists of global settings and one or more `[server NAME]` sections.
+- Relative paths are resolved relative to the config file location.
+
+**Global settings:**
+- `code`: Directory for checked-out and indexed source code. Default: `[workdir]/code`
+- `fileindex`: Path to the file index file. Default: `[workdir]/csearch.fileindex`
+- `index`: Path to the code index file. Default: `[workdir]/csearch.index`
+- `port`: Port for the server. Default: `80`
+- `manifest`: Path to the manifest file. Default: `[workdir]/manifest.json`
+- `workdir`: Working directory managed by the program. Required.
+
+**Server section (`[server NAME]`):**
+- `api`: GitHub REST API URL. Default: `https://api.github.com`
+- `exclude`: Regex to exclude repositories (at most one).
+- `include`: Repository or owner to include. Formats:
+    - `OWNER` (all repos for user/org)
+    - `OWNER/REPO` (specific repo)
+    - `OWNER/REPO#BRANCH` (specific branch)
+    - `OWNER/REPO#REF` (specific commit)
+- `token`: OAuth2 token (e.g., personal access token).
+- `weburl`: Git web interface URL. Default: `https://github.com`
+- `url`: Base URL for cloning (e.g., `git@github.com`, `https://github.com`). Required.
+
+**Example config:**
+```ini
+[global]
+code = db
+index = /home/user/.csearchindex
+port = 8080
+
+[server github]
+exclude = ^DEPRECATED
+include = freva
+include = torvalds/linux  
+
+[server internal]
+api = https://git.example.com/api
+include = internalorg
+token = ghp_YYYYYYYYYYYYYYYYYYYY
+weburl = https://git.example.com
+url = https://ghp_YYYYYYYYYYYYYYYYYYYY@git.example.com
+```
+
+## Binaries
+
+The following binaries are produced:
+- `cgrep`: Command-line code search using the index. Usage:
+  ```sh
+  cgrep [flags] regexp [file...]
+  ```
+  Flags: `-c` (count), `-h` (no filename), `-i` (case-insensitive), `-l` (list files), `-n` (line numbers), `-v` (invert match)
+
+- `cindex`: Builds the code search index. Usage:
+  ```sh
+  cindex [flags] [path...]
+  ```
+  Flags: `-index` (index file), `-list` (list indexed paths), `-reset` (discard existing index), `-zip` (index zip files)
+
+- `csearch`: Behaves like `cgrep`, but over all indexed files with option to limit search to files matching a regex. Usage:
+  ```sh
+  csearch [flags] regexp
+  ```
+  Flags: `-c` (count), `-f` (file regexp), `-h` (no filename), `-i` (case-insensitive), `-l` (list files), `-n` (line numbers), `-index` (index file)
+
+- `cserver`: HTTP server providing the web UI. Usage:
+  ```sh
+  cserver --config path/to/config
+  ```
+- `csupdater`: Updates repositories and indices from remote sources. Usage:
+  ```sh
+  csupdater --config path/to/config [--manifest] [--sync] [--index] [--verbose] [--exit-early] [--help-config]
+  ```
+  Flags: `--config` (config file), `--exit-early` (exit without updating index if no change in sync), `--manifest` (whether to update manifest), `--sync` (whether to sync repositories), `--index` (whether to update index)
+  By default, runs all update steps.
+
+## Building
+
+Requirements: [Go](https://golang.org/doc/install) (>=1.23), [pnpm](https://pnpm.io/) (>=10).
+
+Run:
+```sh
+make
+```
+This installs Go binaries and builds the frontend UI.
+
+## Automatic Updates and Server
+
+To run the server and keep the index up to date automatically, install the provided systemd user services:
+
+1. Copy the service files from `systemd/` to your user systemd directory:
+   ```sh
+   cp systemd/codesearch-server.service systemd/codesearch-updater.service ~/.config/systemd/user/
+   ```
+   then edit them to set the correct paths for the binaries and config file.
+2. Reload systemd and enable the services:
+   ```sh
+   systemctl --user daemon-reload
+   systemctl --user enable --now codesearch-server.service codesearch-updater.service
+   ```
+
+- `codesearch-server.service` runs the HTTP server (`cserver`).
+- `codesearch-updater.service` keeps the repositories and index up to date (`csupdater`).

Makefile

@@ -11,7 +11,7 @@ import (
 	"os"
 	"runtime/pprof"
 
-	"github.com/google/codesearch/regexp"
+	"github.com/freva/codesearch/regexp"
 )
 
 var usageMessage = `usage: cgrep [-c] [-h] [-i] [-l] [-n] [-v] regexp [file...]
@@ -24,7 +24,7 @@ cannot be abbreviated to -in.
 `
 
 func usage() {
-	fmt.Fprintf(os.Stderr, usageMessage)
+	fmt.Fprint(os.Stderr, usageMessage)
 	os.Exit(2)
 }
 

README

@@ -13,13 +13,13 @@ import (
 	"runtime/pprof"
 	"slices"
 
-	"github.com/google/codesearch/index"
+	"github.com/freva/codesearch/index"
 )
 
-var usageMessage = `usage: cindex [-list] [-reset] [-zip] [path...]
+var usageMessage = `usage: cindex [-index file] [-list] [-reset] [-zip] [path...]
 
 Cindex prepares the trigram index for use by csearch.  The index is the
-file named by $CSEARCHINDEX, or else $HOME/.csearchindex.
+file named by -index, or else $CSEARCHINDEX, or else $HOME/.csearchindex.
 
 The simplest invocation is
 
@@ -53,11 +53,12 @@ With no path arguments, cindex -reset removes the index.
 `
 
 func usage() {
-	fmt.Fprintf(os.Stderr, usageMessage)
+	fmt.Fprint(os.Stderr, usageMessage)
 	os.Exit(2)
 }
 
 var (
+	indexFlag   = flag.String("index", "", "path to index file")
 	listFlag    = flag.Bool("list", false, "list indexed paths and exit")
 	resetFlag   = flag.Bool("reset", false, "discard existing index")
 	verboseFlag = flag.Bool("verbose", false, "print extra information")
@@ -72,8 +73,10 @@ func main() {
 	flag.Usage = usage
 	flag.Parse()
 
+	indexpath := index.File(*indexFlag)
+
 	if *listFlag {
-		ix := index.Open(index.File())
+		ix := index.Open(index.File(indexpath))
 		if *checkFlag {
 			if err := ix.Check(); err != nil {
 				log.Fatal(err)
@@ -96,12 +99,12 @@ func main() {
 	}
 
 	if *resetFlag && flag.NArg() == 0 {
-		os.Remove(index.File())
+		os.Remove(index.File(indexpath))
 		return
 	}
 	var roots []index.Path
 	if flag.NArg() == 0 {
-		ix := index.Open(index.File())
+		ix := index.Open(index.File(indexpath))
 		roots = slices.Collect(ix.Roots().All())
 	} else {
 		// Translate arguments to absolute paths so that
@@ -117,7 +120,7 @@ func main() {
 		slices.SortFunc(roots, index.Path.Compare)
 	}
 
-	master := index.File()
+	master := index.File(indexpath)
 	if _, err := os.Stat(master); err != nil {
 		// Does not exist.
 		*resetFlag = true

README.md

@@ -14,8 +14,8 @@ import (
 	"runtime/pprof"
 	"strings"
 
-	"github.com/google/codesearch/index"
-	"github.com/google/codesearch/regexp"
+	"github.com/freva/codesearch/index"
+	"github.com/freva/codesearch/regexp"
 )
 
 var usageMessage = `usage: csearch [-c] [-f fileregexp] [-h] [-i] [-l] [-n] regexp
@@ -44,14 +44,15 @@ empty, $HOME/.csearchindex.
 `
 
 func usage() {
-	fmt.Fprintf(os.Stderr, usageMessage)
+	fmt.Fprint(os.Stderr, usageMessage)
 	os.Exit(2)
 }
 
 var (
 	fFlag       = flag.String("f", "", "search only files with names matching this regexp")
 	iFlag       = flag.Bool("i", false, "case-insensitive search")
 	htmlFlag    = flag.Bool("html", false, "print HTML output")
+	indexFlag   = flag.String("index", "", "path to index file")
 	verboseFlag = flag.Bool("verbose", false, "print extra information")
 	bruteFlag   = flag.Bool("brute", false, "brute force - search all files in index")
 	cpuProfile  = flag.String("cpuprofile", "", "write cpu profile to this file")
@@ -109,7 +110,7 @@ func Main() {
 		log.Printf("query: %s\n", q)
 	}
 
-	ix := index.Open(index.File())
+	ix := index.Open(index.File(*indexFlag))
 	ix.Verbose = *verboseFlag
 	var post []int
 	if *bruteFlag {