diff --git a/bin/process-docs.sh b/bin/process-docs.sh index 7236faad887..f155f7aaa77 100755 --- a/bin/process-docs.sh +++ b/bin/process-docs.sh @@ -18,96 +18,173 @@ # under the License. # -pushd "$(dirname $0)/.." > /dev/null +# Builds TinkerPop documentation using the gremlin-docs AsciidoctorJ extension. +# The extension delegates Gremlin code execution to a real Gremlin Console process, +# then generates language variant tabs via the ANTLR-based GremlinTranslator. +# +# Usage: +# bin/process-docs.sh # full build with live gremlin execution +# bin/process-docs.sh --dry-run # skip gremlin execution (fast, for layout checks) -NOCLEAN= +set -e -DRYRUN= -DRYRUN_DOCS= -FULLRUN_DOCS= +PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "${PROJECT_ROOT}" -makeAbsPaths () { - for doc in $(tr ',' $'\n' <<< "$1"); do - if [ -d $doc ]; then - for d in $(find "$doc" -name "*.asciidoc"); do - echo $(cd $(dirname "$d") && pwd -P)/$(basename "$d") - done - else - echo $(cd $(dirname "$doc") && pwd -P)/$(basename "$doc") - fi - done | paste -sd ',' - -} +TP_VERSION=$(cat pom.xml | grep -A1 'tinkerpop' | grep '' | sed -e 's/.*//' -e 's/<\/version>.*//') -while [[ $# -gt 0 ]] -do - key="$1" - case $key in - -n|--noClean) - NOCLEAN=1 - shift - ;; - -d|--dryRun) - DRYRUN=1 - shift - if [[ $# -gt 0 ]] && [[ $1 != -* ]]; then - DRYRUN_DOCS=$(makeAbsPaths "$1") - shift - else - DRYRUN_DOCS="*" - fi - ;; - -f|--fullRun) - DRYRUN=1 - DRYRUN_DOCS=${DRYRUN_DOCS:-"*"} - shift - FULLRUN_DOCS=$(makeAbsPaths "$1") - shift - ;; - *) - # unknown option - shift - ;; - esac -done +if [ -z "${TP_VERSION}" ]; then + echo "ERROR: Could not determine TinkerPop version from pom.xml" + exit 1 +fi -if [ -z ${NOCLEAN} ]; then - rm -rf ~/.groovy/grapes/org.apache.tinkerpop/ - if hash hadoop 2> /dev/null; then - hadoop fs -rm -r "hadoop-gremlin-*-libs" > /dev/null 2>&1 - fi +ASCIIDOC_ATTRS="" +DRYRUN=false +if [ "$1" = "--dry-run" ]; then + DRYRUN=true + ASCIIDOC_ATTRS="-Dasciidoctor.attributes.gremlin-docs-dryrun=true" + echo "Dry-run mode: gremlin blocks will not be executed" fi -if [ ${DRYRUN} ] && [ "${DRYRUN_DOCS}" == "*" ] && [ -z "${FULLRUN_DOCS}" ]; then +echo "Building docs for TinkerPop ${TP_VERSION}..." +echo "Source: docs/src/" +echo "Output: target/docs/htmlsingle/" + +# build and install the gremlin-docs extension (not part of the main reactor) +echo "Installing gremlin-docs extension..." +mvn install -f gremlin-docs/pom.xml -DskipTests -Denforcer.skip=true -q - mkdir -p target/postprocess-asciidoc/tmp - cp -R docs/{static,stylesheets} target/postprocess-asciidoc/ - cp -R docs/src/. target/postprocess-asciidoc/ - ec=$? +GREMLIN_SERVER_PID="" +GEPHI_MOCK_PID="" + +function cleanup() { + if [ -n "${GREMLIN_SERVER_PID}" ]; then + echo "Stopping Gremlin Server (PID ${GREMLIN_SERVER_PID})..." + kill ${GREMLIN_SERVER_PID} 2>/dev/null || true + wait ${GREMLIN_SERVER_PID} 2>/dev/null || true + fi + if [ -n "${GEPHI_MOCK_PID}" ]; then + kill ${GEPHI_MOCK_PID} 2>/dev/null || true + wait ${GEPHI_MOCK_PID} 2>/dev/null || true + fi + # clean up conf/hadoop from console home if we created it + if [ -n "${CONSOLE_HOME}" ] && [ -d "${CONSOLE_HOME}/conf/hadoop" ]; then + rm -rf "${CONSOLE_HOME}/conf/hadoop" + fi +} -else +trap cleanup EXIT - GEPHI_MOCK= +if [ "${DRYRUN}" = "false" ]; then + # locate the console distribution (must be built already via mvn install -pl :gremlin-console -am) + CONSOLE_HOME=$(ls -d "${PROJECT_ROOT}"/gremlin-console/target/apache-tinkerpop-gremlin-console-*-standalone 2>/dev/null | head -1) - trap cleanup EXIT + if [ -z "${CONSOLE_HOME}" ] || [ ! -d "${CONSOLE_HOME}" ]; then + echo "ERROR: Gremlin Console distribution not found." + echo "Build it first: mvn clean install -pl :gremlin-console -am -DskipTests" + exit 1 + fi - function cleanup() { - [ ${GEPHI_MOCK} ] && kill ${GEPHI_MOCK} - } + echo "Using console: ${CONSOLE_HOME}" + + # install plugins needed for doc examples + # NOTE: neo4j-gremlin is excluded by default because its Spark jars conflict with + # spark-gremlin on the classpath. Neo4j examples will fall back to dry-run output. + # The old AWK pipeline handled this by swapping plugins per-document. + PLUGIN_DIR="${CONSOLE_HOME}/ext" + plugins=("hadoop-gremlin" "spark-gremlin" "sparql-gremlin") + for pluginName in "${plugins[@]}"; do + if [ ! -d "${PLUGIN_DIR}/${pluginName}" ]; then + echo "Installing plugin: ${pluginName}..." + pushd "${CONSOLE_HOME}" > /dev/null + bin/gremlin.sh -e <(echo ":install org.apache.tinkerpop ${pluginName} ${TP_VERSION}") 2>/dev/null || true + popd > /dev/null + else + echo "Plugin already installed: ${pluginName}" + fi + done + + # activate plugins in plugins.txt if not already present + for pluginName in "${plugins[@]}"; do + # derive class name: hadoop-gremlin -> HadoopGremlinPlugin + className="" + for part in $(tr '-' '\n' <<< "${pluginName}"); do + className="${className}$(tr '[:lower:]' '[:upper:]' <<< "${part:0:1}")${part:1}" + done + pluginClassFile=$(find . -name "${className}Plugin.java" 2>/dev/null | head -1) + if [ -n "${pluginClassFile}" ]; then + pluginClass=$(sed -e 's@.*src/main/java/@@' -e 's/\.java$//' <<< "${pluginClassFile}" | tr '/' '.') + if ! grep -q "${pluginClass}" "${PLUGIN_DIR}/plugins.txt" 2>/dev/null; then + echo "${pluginClass}" >> "${PLUGIN_DIR}/plugins.txt" + fi + fi + done + + # start Gremlin Server for remote connection examples + SERVER_HOME=$(ls -d "${PROJECT_ROOT}"/gremlin-server/target/apache-tinkerpop-gremlin-server-*-standalone 2>/dev/null | head -1) + if [ -n "${SERVER_HOME}" ] && [ -d "${SERVER_HOME}" ]; then + # check for port conflict before starting + if nc -z localhost 8182 2>/dev/null; then + echo "ERROR: Port 8182 is already in use. Stop the process using it before building docs." + exit 1 + fi + + echo "Starting Gremlin Server..." + mkdir -p target/docs-logs + pushd "${SERVER_HOME}" > /dev/null + bin/gremlin-server.sh conf/gremlin-server-modern.yaml > "${PROJECT_ROOT}/target/docs-logs/gremlin-server.log" 2>&1 & + GREMLIN_SERVER_PID=$! + popd > /dev/null + + # wait for server to be ready (up to 30 seconds) + echo -n "Waiting for Gremlin Server on port 8182" + for i in $(seq 1 30); do + if nc -z localhost 8182 2>/dev/null; then + echo " ready." + break + fi + echo -n "." + sleep 1 + done + if ! nc -z localhost 8182 2>/dev/null; then + echo " WARNING: Gremlin Server may not have started. Remote connection examples may fail." + fi + else + echo "WARNING: Gremlin Server distribution not found. Remote connection examples will fail." + echo "Build it first: mvn clean install -pl :gremlin-server -am -DskipTests" + fi - nc -z localhost 8080 || ( - bin/gephi-mock.py > /dev/null 2>&1 & - GEPHI_MOCK=$! - ) + # set up conf/hadoop inside the console home so GraphFactory.open('conf/hadoop/...') resolves + # (the console process runs with CONSOLE_HOME as its working directory) + mkdir -p "${CONSOLE_HOME}/conf/hadoop" + cp "${PROJECT_ROOT}"/hadoop-gremlin/conf/* "${CONSOLE_HOME}/conf/hadoop/" 2>/dev/null || true - docs/preprocessor/preprocess.sh "${DRYRUN_DOCS}" "${FULLRUN_DOCS}" - ec=$? -fi + # start Gephi mock server for Gephi plugin examples (listens on port 8080) + if ! nc -z localhost 8080 2>/dev/null; then + "${PROJECT_ROOT}/bin/gephi-mock.py" > /dev/null 2>&1 & + GEPHI_MOCK_PID=$! + fi -if [ $ec -eq 0 ]; then - mvn process-resources -Dasciidoc && docs/postprocessor/postprocess.sh - ec=$? + HADOOP_LIBS="${CONSOLE_HOME}/ext/tinkergraph-gremlin/lib" + ASCIIDOC_ATTRS="${ASCIIDOC_ATTRS} -Dasciidoctor.attributes.gremlin-docs-console-home=${CONSOLE_HOME}" + ASCIIDOC_ATTRS="${ASCIIDOC_ATTRS} -Dasciidoctor.attributes.gremlin-docs-hadoop-libs=${HADOOP_LIBS}" fi -popd > /dev/null +# copy static assets that live outside docs/src/ into the staging area +mkdir -p target/doc-source +cp -r docs/static target/doc-source/ 2>/dev/null || true +cp -r docs/stylesheets target/doc-source/ 2>/dev/null || true + +# run asciidoctor with the gremlin-docs extension +mvn process-resources \ + -Dasciidoc \ + -Drat.skip=true \ + ${ASCIIDOC_ATTRS} + +# post-process: replace version placeholder +echo "Post-processing: replacing x.y.z with ${TP_VERSION}..." +find target/docs/htmlsingle -name '*.html' | while IFS= read -r f; do + sed "s/x\.y\.z/${TP_VERSION}/g" "$f" > "$f.tmp" && mv "$f.tmp" "$f" +done -exit ${ec} +echo "Done. Output in target/docs/htmlsingle/" diff --git a/docs/postprocessor/postprocess.sh b/docs/postprocessor/postprocess.sh deleted file mode 100755 index e7dca523b84..00000000000 --- a/docs/postprocessor/postprocess.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -pushd "$(dirname $0)/../.." > /dev/null - -TP_VERSION=$(cat pom.xml | grep -A1 'tinkerpop' | grep -o 'version>[^<]*' | grep -o '>.*' | cut -d '>' -f2 | head -n1) - -if [ -d "target/docs" ]; then - - # redirect the GLV Tutorial to reference docs - sed -i "s///" target/docs/htmlsingle/tutorials/gremlin-language-variants/index.html - - find target/docs -name index.html | while read file ; do - awk -f "docs/postprocessor/processor.awk" "${file}" 2>/dev/null \ - | perl -0777 -pe 's/\/\*\n \*\/<\/span>//igs' \ - | sed "s/x\.y\.z/${TP_VERSION}/g" \ - > "${file}.tmp" && mv "${file}.tmp" "${file}" - done -fi - -popd > /dev/null diff --git a/docs/postprocessor/processor.awk b/docs/postprocessor/processor.awk deleted file mode 100644 index fbeadcccb44..00000000000 --- a/docs/postprocessor/processor.awk +++ /dev/null @@ -1,53 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -BEGIN { - firstMatch=1 - styled=0 -} - -/Licensed to the Apache Software Foundation/ { - isHeader=1 -} - -/<\/style>/ { - if (!styled) { - print ".invisible {color: rgba(0,0,0,0); font-size: 0;}" - styled=1 - } -} - -!// { - if (firstMatch || !isHeader) { - print gensub(/()\(([0-9]+)\)(<\/b>)/, - "//\\1\\2\\3", "g") - } -} - -// { - if (firstMatch || !isHeader) { - print gensub(/\/\/<\/span>[ ]*()\(([0-9]+)\)(<\/b>)/, - "//\\1\\2\\3\\\\<\/span>", "g") - } -} - -/under the License\./ { - firstMatch=0 - isHeader=0 -} diff --git a/docs/preprocessor/awk/cleanup.awk b/docs/preprocessor/awk/cleanup.awk deleted file mode 100644 index b49a2cc772c..00000000000 --- a/docs/preprocessor/awk/cleanup.awk +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# @author Daniel Kuppitz (http://gremlin.guru) -# -/^gremlin> '\+EVALUATED'$/ { evaluated = 1 } -/^==>\-EVALUATED$/ { evaluated = 0 } - -!/^((gremlin> ')|==>)[\+\-]EVALUATED(')?$/ { - if ($0 !~ /^gremlin> pb\([0-9]*\); '----'$/) { - if (!evaluated || $0 !~ /^gremlin> [']?:/) { - if (evaluated && $0 ~ /^==>:/) gsub(/^==>/, "gremlin> ") - if (!evaluated || $0 == "==>----") gsub(/^==>/, "") - if (evaluated) { - if ($0 !~ /^WARN /) print - } else if ($0 !~ /^gremlin> pb\([0-9]*\); / && $0 !~ /^gremlin> $/) { - print - } - } - } -} diff --git a/docs/preprocessor/awk/ignore.awk b/docs/preprocessor/awk/ignore.awk deleted file mode 100644 index 99f55e06d90..00000000000 --- a/docs/preprocessor/awk/ignore.awk +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# @author Daniel Kuppitz (http://gremlin.guru) -# -/^gremlin> '\+IGNORE'$/ { ignore = 1 } -{ if (!ignore) print } -/^==>\-IGNORE$/ { ignore = 0 } diff --git a/docs/preprocessor/awk/init-code-blocks.awk b/docs/preprocessor/awk/init-code-blocks.awk deleted file mode 100644 index 8349d6e6410..00000000000 --- a/docs/preprocessor/awk/init-code-blocks.awk +++ /dev/null @@ -1,73 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# @author Daniel Kuppitz (http://gremlin.guru) -# -function capitalize(string) { - return toupper(substr(string, 1, 1)) substr(string, 2) -} - -BEGIN { - delimiter = 0 -} - -/^pb\([0-9]*\); '\[gremlin-/ { - delimiter = 1 - split($0, a, "-") - b = gensub(/]'/, "", "g", a[2]) - split(b, c, ",") - split(a[1], d, ";") - lang = c[1] - graph = c[2] - print d[1] "; '[source," lang "]'" - print "'+EVALUATED'" - print "'+IGNORE'" - if (graph != "existing") { - if (graph) { - print "graph = TinkerFactory.create" capitalize(graph) "()" - } else { - print "graph = TinkerGraph.open()" - } - print "g = graph.traversal()" - print "marko = g.V().has('name', 'marko').tryNext().orElse(null)" - print "f = new File('/tmp/neo4j')" - print "if (f.exists()) f.deleteDir()" - print "f = new File('/tmp/tinkergraph.kryo')" - print "if (f.exists()) f.deleteDir()" - print ":set max-iteration 100" - } - print "'-IGNORE'" -} - -!/^pb\([0-9]*\); '\[gremlin-/ { - if (delimiter == 2 && !($0 ~ /^pb\([0-9]*\); '----'/)) { - switch (lang) { - default: - print - break - } - } else print -} - -/^pb\([0-9]*\); '----'/ { - if (delimiter == 1) delimiter = 2 - else if (delimiter == 2) { - print "'-EVALUATED'" - delimiter = 0 - } -} diff --git a/docs/preprocessor/awk/language-variants.awk b/docs/preprocessor/awk/language-variants.awk deleted file mode 100644 index 9d9fbf03dff..00000000000 --- a/docs/preprocessor/awk/language-variants.awk +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# @author Daniel Kuppitz (http://gremlin.guru) -# -BEGIN { - lang = null - inCodeBlock = 0 -} - -/^\[source,/ { - delimiter = 1 - split($0, a, ",") - lang = gensub(/]$/, "", 1, a[2]) -} - -/^----$/ { - if (inCodeBlock == 0) inCodeBlock = 1 - else inCodeBlock = 0 -} - -{ if (inCodeBlock) { - switch (lang) { - default: - print - break - } - } else print -} diff --git a/docs/preprocessor/awk/prepare.awk b/docs/preprocessor/awk/prepare.awk deleted file mode 100644 index 860dd4ee2bb..00000000000 --- a/docs/preprocessor/awk/prepare.awk +++ /dev/null @@ -1,83 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# @author Daniel Kuppitz (http://gremlin.guru) -# -BEGIN { - p = 0 - c = "//" -} - -function escape_string(string) { - str = gensub(/\\/, "\\\\\\\\", "g", string) - return gensub(/'/, "\\\\'", "g", str) -} - -function print_string(string) { - print "pb(" p++ "); '" escape_string(string) "'" -} - -function transform_callouts(code) { - return gensub(/\s*((<[0-9]+>\s*)*<[0-9]+>)\s*$/, " " c c " \\1", "g", code) -} - -function remove_callouts(code) { - return gensub(/\s*((<[0-9]+>\s*)*<[0-9]+>)\s*$/, "", "g", code) -} - -/^----$/ { - if (inCodeSection) { - if (prepared) { - inCodeSection = 0 - prepared = 0 - } else { - prepared = 1 - } - } - print_string($0) -} - -!/^----$/ { - if (inCodeSection) { - if ($0 ~ /^:/) { - print "'" escape_string(transform_callouts($0)) "'" - print remove_callouts($0) - } else { - print transform_callouts($0) - } - } else { - print_string($0) - } -} - -/^\[gremlin-/ { - inCodeSection = 1 - split($0, a, "-") - b = gensub(/]'/, "", "g", a[2]) - split(b, l, ",") - lang = l[1] - switch (lang) { - default: - c = "//" - break - } -} - -END { - print_string("// LAST LINE") -} diff --git a/docs/preprocessor/awk/prettify.awk b/docs/preprocessor/awk/prettify.awk deleted file mode 100644 index 522ac631924..00000000000 --- a/docs/preprocessor/awk/prettify.awk +++ /dev/null @@ -1,31 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# @author Daniel Kuppitz (http://gremlin.guru) -# -/^==>\/\/\/\/$/ { doPrint = 1 } - -{ - if (inCodeSection && $0 ~ /^\.*[0-9]+> /) { - gsub(/^.{8}/, " ") - } - if (doPrint) print -} - -/^==>\+EVALUATED/ { inCodeSection = 1 } -/^==>\-EVALUATED/ { inCodeSection = 0 } diff --git a/docs/preprocessor/awk/progressbar.awk b/docs/preprocessor/awk/progressbar.awk deleted file mode 100644 index 25109c42e00..00000000000 --- a/docs/preprocessor/awk/progressbar.awk +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# @author Daniel Kuppitz (http://gremlin.guru) -# -BEGIN { - max = 0 - content = "" -} - -/^pb\([0-9]*\); / { - max = gensub(/^pb\(([0-9]*)\); .*/, "\\1", "g", $0) -} -{ content = content "\n" $0 } - -END { - while ((getline line < tpl) > 0) { - print gensub(/TOTAL_LINES/, max, "g", line) - } - print content -} diff --git a/docs/preprocessor/awk/tabify.awk b/docs/preprocessor/awk/tabify.awk deleted file mode 100644 index 7e68c13e5dd..00000000000 --- a/docs/preprocessor/awk/tabify.awk +++ /dev/null @@ -1,132 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# -# @author Daniel Kuppitz (http://gremlin.guru) -# -function print_tabs(next_id, tabs, blocks) { - - num_tabs = length(tabs) - x = next_id - - print "++++" - print "
" - - for (i = 1; i <= num_tabs; i++) { - title = tabs[i] - print " " - print " " - x++ - } - - for (i = 1; i<= num_tabs; i++) { - print "
" - print "
" - print "++++\n" - print blocks[i] - print "++++" - print "
" - print "
" - } - - print "
" - print "++++\n" -} - -function transform_callouts(code, c) { - return gensub(/\s*((<[0-9]+>\s*)*<[0-9]+>)\s*\n/, " " c c " \\1\\2\n", "g", code) -} - -BEGIN { - id_part=systime() - status = 0 - next_id = 1 - block[0] = 0 # initialize "blocks" as an array - delete blocks[0] -} - -/^\[gremlin-/ { - status = 1 - lang = gensub(/^\[gremlin-([^,\]]+).*/, "\\1", "g", $0) - code = "" - evaluate = 1 -} - -/^\[source,(csharp|groovy|java|javascript|python|go),tab\]/ { - status = 1 - lang = gensub(/^\[source,([^,\]]+).*/, "\\1", "g", $0) - code = "" - evaluate = 0 -} - -/^\[source,(csharp|groovy|java|javascript|python|go)\]/ { - if (status == 3) { - status = 1 - lang = gensub(/^\[source,([^\]]+).*/, "\\1", "g", $0) - code = "" - } -} - -! /^\[source,(csharp|groovy|java|javascript|python|go)/ { - if (status == 3 && $0 != "") { - print_tabs(next_id, tabs, blocks) - next_id = next_id + length(tabs) - for (i in tabs) { - delete tabs[i] - delete blocks[i] - } - status = 0 - } -} - -/^----$/ { - if (status == 1) { - status = 2 - } else if (status == 2) { - status = 3 - } -} - -{ if (status == 3) { - if ($0 == "----") { - i = length(blocks) + 1 - if (i == 1 && evaluate == 1) { - tabs[i] = "console (" lang ")" - blocks[i] = code_header code "\n" $0 "\n" - i++ - } - tabs[i] = lang - switch (lang) { - default: - c = "//" - break - } - blocks[i] = "[source," lang "]" transform_callouts(code, c) "\n" $0 "\n" - } - } else { - if (status == 0) print - else if (status == 1) code_header = gensub(/,tab/, "", "g", $0) - else code = code "\n" $0 - } -} - -END { - # EOF - if (status == 3) { - print_tabs(next_id, tabs, blocks) - } -} diff --git a/docs/preprocessor/control-characters.sh b/docs/preprocessor/control-characters.sh deleted file mode 100755 index b8336d6bf01..00000000000 --- a/docs/preprocessor/control-characters.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# The main purpose of this script is to remove control characters -# that are occasionally hidden in the Groovy console's output - -sed -r 's/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' diff --git a/docs/preprocessor/install-plugins.groovy b/docs/preprocessor/install-plugins.groovy deleted file mode 100644 index 0edbfdbd48e..00000000000 --- a/docs/preprocessor/install-plugins.groovy +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/** - * @author Daniel Kuppitz (http://gremlin.guru) - */ -import org.apache.tinkerpop.gremlin.groovy.util.Artifact -import org.apache.tinkerpop.gremlin.groovy.util.DependencyGrabber - -installPlugin = { def artifact -> - def classLoader = new groovy.lang.GroovyClassLoader() - def extensionPath = new File(System.getProperty("user.dir"), "ext") - try { - System.err.print(" * ${artifact.getArtifact()} ... ") - new DependencyGrabber(classLoader, extensionPath).copyDependenciesToPath(artifact) - System.err.println("done") - } catch (Exception e) { - System.err.println("failed") - System.err.println() - System.err.println(e.getMessage()) - e.printStackTrace() - System.exit(1) - } -} - -:plugin use tinkerpop.sugar -:plugin use tinkerpop.credentials -System.err.println("done") diff --git a/docs/preprocessor/install-plugins.sh b/docs/preprocessor/install-plugins.sh deleted file mode 100755 index ba882132076..00000000000 --- a/docs/preprocessor/install-plugins.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -CONSOLE_HOME=$1 -TP_VERSION=$2 -TMP_DIR=$3 -INSTALL_TEMPLATE="docs/preprocessor/install-plugins.groovy" -INSTALL_FILE="${TMP_DIR}/install-plugins.groovy" - -plugins=("hadoop-gremlin" "spark-gremlin" "neo4j-gremlin" "sparql-gremlin") -# plugins=() -pluginsCount=${#plugins[@]} - -i=0 - -cp ${INSTALL_TEMPLATE} ${INSTALL_FILE} - -while [ ${i} -lt ${pluginsCount} ]; do - pluginName=${plugins[$i]} - className="" - for part in $(tr '-' '\n' <<< ${pluginName}); do - className="${className}$(tr '[:lower:]' '[:upper:]' <<< ${part:0:1})${part:1}" - done - pluginClassFile=$(find . -name "${className}Plugin.java") - pluginClass=`sed -e 's@.*src/main/java/@@' -e 's/\.java$//' <<< ${pluginClassFile} | tr '/' '.'` - installed=`grep -c "${pluginClass}" ${CONSOLE_HOME}/ext/plugins.txt` - if [ ${installed} -eq 0 ]; then - echo "installPlugin(new Artifact(\"org.apache.tinkerpop\", \"${pluginName}\", \"${TP_VERSION}\"))" >> ${INSTALL_FILE} - echo "${pluginName}" >> ${TMP_DIR}/plugins.dir - echo "${pluginClass}" >> ${TMP_DIR}/plugins.txt - else - echo " * skipping ${pluginName} (already installed)" - fi - ((i++)) -done - -echo "System.exit(0)" >> ${INSTALL_FILE} -echo -ne " * tinkerpop-sugar ... " - -pushd ${CONSOLE_HOME} > /dev/null - -mkdir -p ~/.java/.userPrefs -chmod 700 ~/.java/.userPrefs - -bin/gremlin.sh -e ${INSTALL_FILE} > /dev/null - -if [ ${PIPESTATUS[0]} -ne 0 ]; then - popd > /dev/null - exit 1 -fi - -if [ -f "${TMP_DIR}/plugins.txt" ]; then - cat ${TMP_DIR}/plugins.txt >> ${CONSOLE_HOME}/ext/plugins.txt -fi - -popd > /dev/null diff --git a/docs/preprocessor/preprocess-file.sh b/docs/preprocessor/preprocess-file.sh deleted file mode 100755 index 9ae43c2f8d0..00000000000 --- a/docs/preprocessor/preprocess-file.sh +++ /dev/null @@ -1,174 +0,0 @@ -#!/bin/bash -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -TP_HOME=`pwd` -CONSOLE_HOME=$1 -AWK_SCRIPTS="${TP_HOME}/docs/preprocessor/awk" - -IFS=',' read -r -a DRYRUN_DOCS <<< "$2" -IFS=',' read -r -a FULLRUN_DOCS <<< "$3" - -dryRun () { - local doc - yes=0 - no=1 - doDryRun=${no} - if [ "${DRYRUN_DOCS}" == "*" ]; then - doDryRun=${yes} - else - for doc in "${DRYRUN_DOCS[@]}"; do - if [ "${doc}" == "$1" ]; then - doDryRun=${yes} - break - fi - done - fi - if [ ${doDryRun} ]; then - for doc in "${FULLRUN_DOCS[@]}"; do - if [ "${doc}" == "$1" ]; then - doDryRun=${no} - break - fi - done - fi - return ${doDryRun} -} - -input=$4 -output=`sed 's@/docs/src/@/target/postprocess-asciidoc/@' <<< "${input}"` - -SKIP= -if dryRun ${input}; then - SKIP=1 -fi - -mkdir -p `dirname ${output}` - -if hash stdbuf 2> /dev/null; then - lb="stdbuf -oL" -else - lb="" -fi - -trap cleanup INT - -function cleanup { - if [ -f "${output}" ]; then - if [ `wc -l "${output}" | awk '{print $1}'` -gt 0 ]; then - echo -e "\n\e[1mLast 10 lines of ${output}:\e[0m\n" - tail -n10 ${output} - echo - echo "Opening ${output} for full inspection" - sleep 5 - less ${output} - fi - fi - rm -rf ${output} ${CONSOLE_HOME}/.ext - exit 255 -} - -function processed { - echo -ne "\r progress: [====================================================================================================] 100%\n" -} - -echo -echo " * source: ${input}" -echo " target: ${output}" -echo -ne " progress: initializing" - -if [ ! ${SKIP} ] && [ $(grep -c '^\[gremlin' ${input}) -gt 0 ]; then - if [ ${output} -nt ${input} ]; then - processed - exit 0 - fi - pushd "${CONSOLE_HOME}" > /dev/null - - doc=`basename ${input} .asciidoc` - - case "${doc}" in - "implementations-neo4j") - # deactivate Spark plugin to prevent version conflicts between TinkerPop's Spark jars and Neo4j's Spark jars - mkdir .ext - mv ext/spark-gremlin .ext/ - cat ext/plugins.txt | tee .ext/plugins.all | grep -Fv 'SparkGremlinPlugin' > .ext/plugins.txt - ;; - "implementations-hadoop-start" | "implementations-hadoop-end" | "implementations-spark" | "olap-spark-yarn") - # deactivate Neo4j plugin to prevent version conflicts between TinkerPop's Spark jars and Neo4j's Spark jars - mkdir .ext - mv ext/neo4j-gremlin .ext/ - cat ext/plugins.txt | tee .ext/plugins.all | grep -Fv 'Neo4jGremlinPlugin' > .ext/plugins.txt - ;; - "gremlin-variants") - # deactivate plugin to prevent version conflicts - mkdir .ext - mv ext/neo4j-gremlin .ext/ - mv ext/spark-gremlin .ext/ - mv ext/hadoop-gremlin .ext/ - cat ext/plugins.txt | tee .ext/plugins.all | grep -v 'Neo4jGremlinPlugin\|SparkGremlinPlugin\|HadoopGremlinPlugin' > .ext/plugins.txt - ;; - esac - - if [ -d ".ext" ]; then - mv .ext/plugins.txt ext/ - fi - - sed 's/\t/ /g' ${input} | - awk -f ${AWK_SCRIPTS}/tabify.awk | - awk -f ${AWK_SCRIPTS}/prepare.awk | - awk -f ${AWK_SCRIPTS}/init-code-blocks.awk -v TP_HOME="${TP_HOME}" | - awk -f ${AWK_SCRIPTS}/progressbar.awk -v tpl=${AWK_SCRIPTS}/progressbar.groovy.template | - HADOOP_GREMLIN_LIBS="${CONSOLE_HOME}/ext/tinkergraph-gremlin/lib" bin/gremlin.sh | ${TP_HOME}/docs/preprocessor/control-characters.sh | - ${lb} awk -f ${AWK_SCRIPTS}/ignore.awk | - ${lb} awk -f ${AWK_SCRIPTS}/prettify.awk | - ${lb} awk -f ${AWK_SCRIPTS}/cleanup.awk | - ${lb} awk -f ${AWK_SCRIPTS}/language-variants.awk > ${output} - - # check exit code for each of the previously piped commands - ps=(${PIPESTATUS[@]}) - for i in {0..9}; do - ec=${ps[i]} - [ ${ec} -eq 0 ] || break - done - - if [ -d ".ext" ]; then - mv .ext/plugins.all ext/plugins.txt - mv .ext/* ext/ - rm -r .ext/ - fi - - if [ ${ec} -eq 0 ]; then - tail -n1 ${output} | grep -F '// LAST LINE' > /dev/null - ec=$? - fi - - if [ ${ec} -eq 0 ]; then - processed - fi - - echo - popd > /dev/null - if [ ${ec} -ne 0 ]; then - cleanup - fi -else - cp ${input} ${output} - processed -fi diff --git a/docs/preprocessor/preprocess.sh b/docs/preprocessor/preprocess.sh deleted file mode 100755 index 3c05d946e63..00000000000 --- a/docs/preprocessor/preprocess.sh +++ /dev/null @@ -1,155 +0,0 @@ -#!/bin/bash -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -DRYRUN_DOCS="$1" -FULLRUN_DOCS="$2" - -pushd "$(dirname $0)/../.." > /dev/null - -if [ "${DRYRUN_DOCS}" != "*" ]; then - - if [ ! -f bin/gremlin.sh ]; then - echo "Gremlin REPL is not available. Cannot preprocess AsciiDoc files." - popd > /dev/null - exit 1 - fi - - for daemon in "NameNode" "DataNode" "ResourceManager" "NodeManager" - do - running=`jps | cut -d ' ' -f2 | grep -c ${daemon}` - if [ ${running} -eq 0 ]; then - echo "Hadoop is not running, be sure to start it before processing the docs." - exit 1 - fi - done - - netstat -an | awk '{print $4}' | grep -o '[0-9]*$' | grep '\b8182\b' > /dev/null && { - echo "The port 8182 is required for Gremlin Server, but it is already in use. Be sure to close the application that currently uses the port before processing the docs." - exit 1 - } - - if [ -e /tmp/neo4j ]; then - echo "The directory '/tmp/neo4j' is required by the pre-processor, be sure to delete it before processing the docs." - exit 1 - fi - - if [ -e /tmp/tinkergraph.kryo ]; then - echo "The file '/tmp/tinkergraph.kryo' is required by the pre-processor, be sure to delete it before processing the docs." - exit 1 - fi -fi - -function directory { - d1=`pwd` - cd $1 - d2=`pwd` - cd $d1 - echo "$d2" -} - -mkdir -p target/postprocess-asciidoc/tmp -mkdir -p target/postprocess-asciidoc/logs -cp -R docs/{static,stylesheets} target/postprocess-asciidoc/ - -TP_HOME=`pwd` -CONSOLE_HOME=`directory "${TP_HOME}/gremlin-console/target/apache-tinkerpop-gremlin-console-*-standalone"` -PLUGIN_DIR="${CONSOLE_HOME}/ext" -TP_VERSION=$(cat pom.xml | grep -A1 'tinkerpop' | grep -o 'version>[^<]*' | grep -o '>.*' | cut -d '>' -f2 | head -n1) -TMP_DIR="/tmp/tp-docs-preprocessor" - -mkdir -p "${TMP_DIR}" - -HISTORY_FILE=".gremlin_groovy_history" -[ -f ~/${HISTORY_FILE} ] && cp ~/${HISTORY_FILE} ${TMP_DIR} - -pushd gremlin-server/target/apache-tinkerpop-gremlin-server-*-standalone > /dev/null -bin/gremlin-server.sh conf/gremlin-server-modern.yaml > ${TP_HOME}/target/postprocess-asciidoc/logs/gremlin-server.log 2>&1 & -GREMLIN_SERVER_PID=$! -popd > /dev/null - -function cleanup() { - echo -ne "\r\n\n" - docs/preprocessor/uninstall-plugins.sh "${CONSOLE_HOME}" "${TMP_DIR}" - [ -f ${TMP_DIR}/plugins.txt.orig ] && mv ${TMP_DIR}/plugins.txt.orig ${CONSOLE_HOME}/ext/plugins.txt - find ${TP_HOME}/docs/src/ -name "*.asciidoc.groovy" | xargs rm -f - [ -f ${TMP_DIR}/${HISTORY_FILE} ] && mv ${TMP_DIR}/${HISTORY_FILE} ~/ - rm -rf ${TMP_DIR} - kill ${GREMLIN_SERVER_PID} &> /dev/null - popd &> /dev/null -} - -trap cleanup EXIT - -if [ "${DRYRUN_DOCS}" != "*" ] || [ ! -z "${FULLRUN_DOCS}" ]; then - - # install plugins - echo - echo "==========================" - echo "+ Installing Plugins +" - echo "==========================" - echo - cp ${CONSOLE_HOME}/ext/plugins.txt ${TMP_DIR}/plugins.txt.orig - docs/preprocessor/install-plugins.sh "${CONSOLE_HOME}" "${TP_VERSION}" "${TMP_DIR}" - - if [ $? -ne 0 ]; then - exit 1 - else - echo - fi - -fi - -# process *.asciidoc files -COLS=${COLUMNS} -[[ ${COLUMNS} -lt 240 ]] && stty cols 240 - -tput rmam - -echo -echo "============================" -echo "+ Processing AsciiDocs +" -echo "============================" - -ec=0 -for subdir in $(find "${TP_HOME}/docs/src/" -name index.asciidoc | xargs -n1 dirname) -do - find "${subdir}" -maxdepth 1 -name "*.asciidoc" | - xargs -n1 basename | - xargs -n1 -I {} echo "echo -ne {}' '; (grep -n {} ${subdir}/index.asciidoc || echo 0) | head -n1 | cut -d ':' -f1" | /bin/bash | sort -nk2 | cut -d ' ' -f1 | - xargs -n1 -I {} echo "${subdir}/{}" | - xargs -n1 ${TP_HOME}/docs/preprocessor/preprocess-file.sh "${CONSOLE_HOME}" "${DRYRUN_DOCS}" "${FULLRUN_DOCS}" - - ps=(${PIPESTATUS[@]}) - for i in {0..7}; do - ec=${ps[i]} - [ ${ec} -eq 0 ] || break - done - [ ${ec} -eq 0 ] || break -done - -tput smam -[[ "${COLUMNS}" != "" ]] && stty cols ${COLS} - -rm -rf /tmp/neo4j /tmp/tinkergraph.kryo - -[ ${ec} -eq 0 ] || exit 1 - -echo diff --git a/docs/preprocessor/uninstall-plugins.sh b/docs/preprocessor/uninstall-plugins.sh deleted file mode 100755 index 6353fe50f71..00000000000 --- a/docs/preprocessor/uninstall-plugins.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -CONSOLE_HOME=$1 -TMP_DIR=$2 - -if [ -f "${TMP_DIR}/plugins.dir" ]; then - for pluginDirectory in $(cat ${TMP_DIR}/plugins.dir); do - rm -rf ${CONSOLE_HOME}/ext/${pluginDirectory} - done -fi - -if [ -f "${TMP_DIR}/plugins.txt" ]; then - for className in $(cat ${TMP_DIR}/plugins.txt); do - sed -e "/${className}/d" ${CONSOLE_HOME}/ext/plugins.txt > ${CONSOLE_HOME}/ext/plugins.txt. - mv ${CONSOLE_HOME}/ext/plugins.txt. ${CONSOLE_HOME}/ext/plugins.txt - done -fi diff --git a/docs/src/dev/developer/development-environment.asciidoc b/docs/src/dev/developer/development-environment.asciidoc index 590c1876dce..9f3a94e27e5 100644 --- a/docs/src/dev/developer/development-environment.asciidoc +++ b/docs/src/dev/developer/development-environment.asciidoc @@ -120,111 +120,19 @@ an issue when working with SNAPSHOT dependencies. [[documentation-environment]] === Documentation Environment -The documentation generation process is not Maven-based and uses shell scripts to process the project's asciidoc. The -scripts should work on Mac and Linux. Javadocs should be built using Java 11. +Documentation is generated using the `gremlin-docs` AsciidoctorJ extension, which delegates Gremlin code block +execution to a real Gremlin Console process and generates language variant tabs via the ANTLR-based +`GremlinTranslator`. The scripts should work on Mac and Linux. -TIP: We recommend performing documentation generation on Linux. For the scripts to work on Mac, you will need to -install GNU versions of the utility programs via `homebrew`, e.g.`grep`, `awk`, `sed`, `findutils`, and `diffutils`. - -To generate documentation, it is required that link:https://hadoop.apache.org[Hadoop 3.3.x] is running in -link:https://hadoop.apache.org/docs/r3.3.1/hadoop-project-dist/hadoop-common/SingleCluster.html#Pseudo-Distributed_Operation[pseudo-distributed] -mode. Be sure to set the `HADOOP_GREMLIN_LIBS` environment variable as described in the -link:https://tinkerpop.apache.org/docs/x.y.z/reference/#hadoop-gremlin[reference documentation]. It is also important -to set the `CLASSPATH` to point at the directory containing the Hadoop configuration files, like `mapred-site.xml`. - -The `/etc/hadoop/yarn-site.xml` file prefers this configuration over the one provided in the Hadoop documentation -referenced above: - -[source,xml] ----- - - - yarn.nodemanager.aux-services - mapreduce_shuffle - - - yarn.nodemanager.vmem-check-enabled - false - - - yarn.nodemanager.vmem-pmem-ratio - 4 - - ----- - -The `/etc/hadoop/mapred-site.xml` file prefers the following configuration: - -[source,xml] ----- - - - mapreduce.framework.name - yarn - - - mapred.map.tasks - 4 - - - mapred.reduce.tasks - 4 - - - mapreduce.job.counters.limit - 1000 - - - mapreduce.jobtracker.address - localhost:9001 - - - mapreduce.map.memory.mb - 2048 - - - mapreduce.reduce.memory.mb - 4096 - - - mapreduce.map.java.opts - -Xmx2048m - - - mapreduce.reduce.java.opts - -Xmx4096m - - ----- - -Also note that link:http://www.grymoire.com/Unix/Awk.html[awk] version `4.0.1` is required for documentation generation. -The link:https://tinkerpop.apache.org/docs/x.y.z/recipes/#olap-spark-yarn[YARN recipe] also uses the `zip` program to -create an archive so that needs to be installed, too, if you don't have it already. - -The Hadoop 3.3.x installation instructions call for installing `pdsh` but installing that seems to cause permission -problems when executing `sbin/start-dfs.sh`. Skipping that prerequisite seems to solve the problem. - -Documentation can be generated locally with: +For a full build with live Gremlin execution, the Gremlin Console and Gremlin Server distributions must be built first: [source,text] +mvn clean install -pl :gremlin-server,:gremlin-console -am -DskipTests bin/process-docs.sh -Documentation is generated to the `target/docs` directory. It is also possible to generate documentation locally with -Docker. `docker/build.sh -d`. - -NOTE: The installation of plugins sometimes fails in this step with the error: `Error grabbing grapes - download -failed`. It often helps in this case to delete the directories for the dependencies that cannot be downloaded -in the `.m2` (`~/.m2/`) and in the `grapes` (`~/.groovy/grapes/`) cache. E.g., if the error is about -`asm#asm;3.2!asm.jar`, then remove the `asm/asm` sub directory in both directories. - -NOTE: Unexpected failures with OLAP often point to a jar conflict that arises in scenarios where Hadoop or Spark -dependencies (or other dependencies for that matter) are modified and conflict. It is not picked up by the enforcer -plugin because the inconsistency arises through plugin installation in Gremlin Console at document generation time. -Making adjustments to the various paths by way of the `` on the jar given the functionality provided -by the `DependencyGrabber` class which allows you to manipulate (typically deleting conflicting files from `/lib` and -`/plugin`) plugin loading will usually resolve it, though it could also be a more general environmental problem with -Spark or Hadoop. The easiest way to see the error is to simply run the examples in the Gremlin Console which more -plainly displays the error than the failure of the documentation generation process. +Documentation is generated to the `target/docs` directory. Use `bin/process-docs.sh --dry-run` to skip Gremlin +execution for faster builds when only checking layout — this mode does not require the Console or Server distributions. +It is also possible to generate documentation locally with Docker. `docker/build.sh -d`. To generate the web site locally, there is no need for any of the above infrastructure. Site generation is a simple shell script: @@ -528,10 +436,7 @@ mvn -Dmaven.javadoc.skip=true --projects tinkergraph-gremlin test * Start Gremlin Server with Docker using the standard test configuration: `docker/gremlin-server.sh` * Check license headers are present: `mvn apache-rat:check` * Build AsciiDocs (see <>): `bin/process-docs.sh` -** Build AsciiDocs (but don't evaluate code blocks): `bin/process-docs.sh --dryRun` -** Build AsciiDocs (but don't evaluate code blocks in specific files): `bin/process-docs.sh --dryRun docs/src/reference/the-graph.asciidoc,docs/src/tutorial/getting-started,...` -** Build AsciiDocs (but evaluate code blocks only in specific files): `bin/process-docs.sh --fullRun docs/src/reference/the-graph.asciidoc,docs/src/tutorial/getting-started,...` -** Process a single AsciiDoc file: +pass:[docs/preprocessor/preprocess-file.sh `pwd`/gremlin-console/target/apache-tinkerpop-gremlin-console-*-standalone "" "*" `pwd`/docs/src/xyz.asciidoc]+ +** Build AsciiDocs (but don't evaluate code blocks): `bin/process-docs.sh --dry-run` * Build JavaDocs/JSDoc: `mvn process-resources -Djavadoc` ** Javadoc to `target/site/apidocs` directory ** JSDoc to the `gremlin-javascript/src/main/javascript/gremlin-javascript/doc/` directory diff --git a/docs/src/dev/developer/for-committers.asciidoc b/docs/src/dev/developer/for-committers.asciidoc index c5e75ebfe08..e169d83712f 100644 --- a/docs/src/dev/developer/for-committers.asciidoc +++ b/docs/src/dev/developer/for-committers.asciidoc @@ -941,9 +941,8 @@ of the Apache "Licensing How-to" for more information. The documentation for TinkerPop is stored in the git repository in `docs/src/` and are then split into several subdirectories, each representing a "book" (or its own publishable body of work). If a new AsciiDoc file is added to -a book, then it should also be included in the `index.asciidoc` file for that book, otherwise the preprocessor will -ignore it. Likewise, if a whole new book (subdirectory) is added, it must include an `index.asciidoc` file to be -recognized by the AsciiDoc preprocessor. +a book, then it should also be included in the `index.asciidoc` file for that book. Likewise, if a whole new book +(subdirectory) is added, it must include an `index.asciidoc` file. Adding a book also requires a change to the root `pom.xml` file. Find the "asciidoc" Maven profile and add a new `` to the `asciidoctor-maven-plugin` configuration. For each book in `docs/src/`, there should be a diff --git a/docs/src/recipes/index.asciidoc b/docs/src/recipes/index.asciidoc index 5537c53e15e..8214dac1efa 100644 --- a/docs/src/recipes/index.asciidoc +++ b/docs/src/recipes/index.asciidoc @@ -123,25 +123,12 @@ included in the `index.asciidoc` with an entry like this: `include::my-recipe.as Documentation should be generated locally for review prior to submitting a pull request. TinkerPop documentation is "live" in that it is bound to a specific version when generated. Furthermore, code examples (those that are `gremlin-groovy` based) are executed at document generation time with the results written directly into the output. -The following command will generate the documentation with: +The following command will generate the documentation: [source,shell] bin/process-docs.sh -The generated documentation can be found at `target/docs/htmlsingle/recipes`. This process can be long on the first -run of the documentation as it is generating all of the documentation locally (e.g. reference documentation, -tutorials, etc). To generate just the recipes, follow this process: - -[source,shell] -bin/process-docs.sh -f docs/src/recipes - -The `bin/process-docs.sh` approach requires that Hadoop is installed. To avoid that prerequisite, try using Docker: - -[source,shell] -docker/build.sh -d - -The downside to using Docker is that the process will take longer as each run will require the entire documentation set -to be generated. +The generated documentation can be found at `target/docs/htmlsingle/recipes`. The final step to submitting a recipe is to issue a link:https://help.github.com/articles/using-pull-requests/[pull request through GitHub]. It is helpful to prefix the name of the pull request with the JIRA issue number, so that TinkerPop's automation between diff --git a/docs/stylesheets/tinkerpop.css b/docs/stylesheets/tinkerpop.css index 71cc47eeec0..b763310c035 100644 --- a/docs/stylesheets/tinkerpop.css +++ b/docs/stylesheets/tinkerpop.css @@ -692,4 +692,4 @@ table.tableblock.grid-all th.tableblock, table.tableblock.grid-all td.tableblock #footer { background-color: #465158; padding: 2em; } #footer-text { color: #eee; font-size: 0.8em; text-align: center; } -.tabs{position:relative;margin:40px auto;width:1024px;max-width:100%;overflow:hidden;padding-top:10px;margin-bottom:60px}.tabs input{position:absolute;z-index:1000;height:50px;left:0;top:0;opacity:0;-ms-filter:"progid:DXImageTransform.Microsoft.Alpha(Opacity=0)";filter:alpha(opacity=0);cursor:pointer;margin:0}.tabs input:hover+label{background:#e08f24}.tabs label{background:#e9ffe9;color:#1a1a1a;font-size:15px;line-height:50px;height:60px;position:relative;top:0;padding:0 20px;float:left;display:block;letter-spacing:1px;text-transform:uppercase;font-weight:bold;text-align:center;box-shadow:2px 0 2px rgba(0,0,0,0.1),-2px 0 2px rgba(0,0,0,0.1);box-sizing:border-box;-webkit-transition:all 150ms ease 0s;transition:all 150ms ease 0s}.tabs label:hover{cursor:pointer}.tabs label:after{content:'';background:#609060;position:absolute;bottom:-2px;left:0;width:100%;height:2px;display:block}.tabs-2 input{width:50%}.tabs-2 input.tab-selector-1{left:0%}.tabs-2 input.tab-selector-2{left:50%}.tabs-2 label{width:50%}.tabs-3 input{width:33.3333333333%}.tabs-3 input.tab-selector-1{left:0%}.tabs-3 input.tab-selector-2{left:33.3333333333%}.tabs-3 input.tab-selector-3{left:66.6666666667%}.tabs-3 label{width:33.3333333333%}.tabs-4 input{width:25%}.tabs-4 input.tab-selector-1{left:0%}.tabs-4 input.tab-selector-2{left:25%}.tabs-4 input.tab-selector-3{left:50%}.tabs-4 input.tab-selector-4{left:75%}.tabs-4 label{width:25%}.tabs-5 input{width:20%}.tabs-5 input.tab-selector-1{left:0%}.tabs-5 input.tab-selector-2{left:20%}.tabs-5 input.tab-selector-3{left:40%}.tabs-5 input.tab-selector-4{left:60%}.tabs-5 input.tab-selector-5{left:80%}.tabs-5 label{width:20%}.tabs-6 input{width:16.6666666667%}.tabs-6 input.tab-selector-1{left:0%}.tabs-6 input.tab-selector-2{left:16.6666666667%}.tabs-6 input.tab-selector-3{left:33.3333333333%}.tabs-6 input.tab-selector-4{left:50%}.tabs-6 input.tab-selector-5{left:66.6666666667%}.tabs-6 input.tab-selector-6{left:83.3333333333%}.tabs-6 label{width:16.6666666667%}.tabs-7 input{width:14.2857142857%}.tabs-7 input.tab-selector-1{left:0%}.tabs-7 input.tab-selector-2{left:14.2857142857%}.tabs-7 input.tab-selector-3{left:28.5714285714%}.tabs-7 input.tab-selector-4{left:42.8571428571%}.tabs-7 input.tab-selector-5{left:57.1428571429%}.tabs-7 input.tab-selector-6{left:71.4285714286%}.tabs-7 input.tab-selector-7{left:85.7142857143%}.tabs-7 label{width:14.2857142857%}.tabs label:first-of-type{z-index:4}.tab-label-2{z-index:4}.tab-label-3{z-index:3}.tab-label-4{z-index:2}.tabs input:checked+label{background:#609060;color:#fefefe;z-index:6}.clear-shadow{clear:both}.tabcontent{height:auto;width:100%;float:left;position:relative;z-index:5;background:#eee;top:-10px;box-sizing:border-box}.tabcontent>div{position:relative;float:left;width:0;height:0;box-sizing:border-box;top:0;left:0;z-index:1;opacity:0;background:#eee}.tabcontent .CodeRay{background-color:#fefefe}.tabs .tab-selector-1:checked ~ .tabcontent .tabcontent-1{z-index:100;-ms-filter:"progid:DXImageTransform.Microsoft.Alpha(Opacity=100)";filter:alpha(opacity=100);opacity:1;width:100%;height:auto;width:100%;height:auto;padding-top:30px}.tabs .tab-selector-2:checked ~ .tabcontent .tabcontent-2{z-index:100;-ms-filter:"progid:DXImageTransform.Microsoft.Alpha(Opacity=100)";filter:alpha(opacity=100);opacity:1;width:100%;height:auto;width:100%;height:auto;padding-top:30px}.tabs .tab-selector-3:checked ~ .tabcontent .tabcontent-3{z-index:100;-ms-filter:"progid:DXImageTransform.Microsoft.Alpha(Opacity=100)";filter:alpha(opacity=100);opacity:1;width:100%;height:auto;width:100%;height:auto;padding-top:30px}.tabs .tab-selector-4:checked ~ .tabcontent .tabcontent-4{z-index:100;-ms-filter:"progid:DXImageTransform.Microsoft.Alpha(Opacity=100)";filter:alpha(opacity=100);opacity:1;width:100%;height:auto;width:100%;height:auto;padding-top:30px}.tabs .tab-selector-5:checked ~ .tabcontent .tabcontent-5{z-index:100;-ms-filter:"progid:DXImageTransform.Microsoft.Alpha(Opacity=100)";filter:alpha(opacity=100);opacity:1;width:100%;height:auto;width:100%;height:auto;padding-top:30px}.tabs .tab-selector-6:checked ~ .tabcontent .tabcontent-6{z-index:100;-ms-filter:"progid:DXImageTransform.Microsoft.Alpha(Opacity=100)";filter:alpha(opacity=100);opacity:1;width:100%;height:auto;width:100%;height:auto;padding-top:30px}.tabs .tab-selector-7:checked ~ .tabcontent .tabcontent-7{z-index:100;-ms-filter:"progid:DXImageTransform.Microsoft.Alpha(Opacity=100)";filter:alpha(opacity=100);opacity:1;width:100%;height:auto;width:100%;height:auto;padding-top:30px} +.tabs{position:relative;margin:40px auto;width:1024px;max-width:100%;overflow:hidden;padding-top:10px;margin-bottom:60px}.tabs input{position:absolute;z-index:1000;height:50px;left:0;top:0;opacity:0;-ms-filter:"progid:DXImageTransform.Microsoft.Alpha(Opacity=0)";filter:alpha(opacity=0);cursor:pointer;margin:0}.tabs input:hover+label{background:#e08f24}.tabs label{background:#e9ffe9;color:#1a1a1a;font-size:15px;line-height:50px;height:60px;position:relative;top:0;padding:0 20px;float:left;display:block;letter-spacing:1px;text-transform:uppercase;font-weight:bold;text-align:center;box-shadow:2px 0 2px rgba(0,0,0,0.1),-2px 0 2px rgba(0,0,0,0.1);box-sizing:border-box;-webkit-transition:all 150ms ease 0s;transition:all 150ms ease 0s}.tabs label:hover{cursor:pointer}.tabs label:after{content:'';background:#609060;position:absolute;bottom:-2px;left:0;width:100%;height:2px;display:block}.tabs-1 input{width:100%}.tabs-1 input.tab-selector-1{left:0%}.tabs-1 label{width:100%}.tabs-2 input{width:50%}.tabs-2 input.tab-selector-1{left:0%}.tabs-2 input.tab-selector-2{left:50%}.tabs-2 label{width:50%}.tabs-3 input{width:33.3333333333%}.tabs-3 input.tab-selector-1{left:0%}.tabs-3 input.tab-selector-2{left:33.3333333333%}.tabs-3 input.tab-selector-3{left:66.6666666667%}.tabs-3 label{width:33.3333333333%}.tabs-4 input{width:25%}.tabs-4 input.tab-selector-1{left:0%}.tabs-4 input.tab-selector-2{left:25%}.tabs-4 input.tab-selector-3{left:50%}.tabs-4 input.tab-selector-4{left:75%}.tabs-4 label{width:25%}.tabs-5 input{width:20%}.tabs-5 input.tab-selector-1{left:0%}.tabs-5 input.tab-selector-2{left:20%}.tabs-5 input.tab-selector-3{left:40%}.tabs-5 input.tab-selector-4{left:60%}.tabs-5 input.tab-selector-5{left:80%}.tabs-5 label{width:20%}.tabs-6 input{width:16.6666666667%}.tabs-6 input.tab-selector-1{left:0%}.tabs-6 input.tab-selector-2{left:16.6666666667%}.tabs-6 input.tab-selector-3{left:33.3333333333%}.tabs-6 input.tab-selector-4{left:50%}.tabs-6 input.tab-selector-5{left:66.6666666667%}.tabs-6 input.tab-selector-6{left:83.3333333333%}.tabs-6 label{width:16.6666666667%}.tabs-7 input{width:14.2857142857%}.tabs-7 input.tab-selector-1{left:0%}.tabs-7 input.tab-selector-2{left:14.2857142857%}.tabs-7 input.tab-selector-3{left:28.5714285714%}.tabs-7 input.tab-selector-4{left:42.8571428571%}.tabs-7 input.tab-selector-5{left:57.1428571429%}.tabs-7 input.tab-selector-6{left:71.4285714286%}.tabs-7 input.tab-selector-7{left:85.7142857143%}.tabs-7 label{width:14.2857142857%}.tabs label:first-of-type{z-index:4}.tab-label-2{z-index:4}.tab-label-3{z-index:3}.tab-label-4{z-index:2}.tabs input:checked+label{background:#609060;color:#fefefe;z-index:6}.clear-shadow{clear:both}.tabcontent{height:auto;width:100%;float:left;position:relative;z-index:5;background:#eee;top:-10px;box-sizing:border-box}.tabcontent>div{position:relative;float:left;width:0;height:0;box-sizing:border-box;top:0;left:0;z-index:1;opacity:0;background:#eee}.tabcontent .CodeRay{background-color:#fefefe}.tabs .tab-selector-1:checked ~ .tabcontent .tabcontent-1{z-index:100;-ms-filter:"progid:DXImageTransform.Microsoft.Alpha(Opacity=100)";filter:alpha(opacity=100);opacity:1;width:100%;height:auto;width:100%;height:auto;padding-top:30px}.tabs .tab-selector-2:checked ~ .tabcontent .tabcontent-2{z-index:100;-ms-filter:"progid:DXImageTransform.Microsoft.Alpha(Opacity=100)";filter:alpha(opacity=100);opacity:1;width:100%;height:auto;width:100%;height:auto;padding-top:30px}.tabs .tab-selector-3:checked ~ .tabcontent .tabcontent-3{z-index:100;-ms-filter:"progid:DXImageTransform.Microsoft.Alpha(Opacity=100)";filter:alpha(opacity=100);opacity:1;width:100%;height:auto;width:100%;height:auto;padding-top:30px}.tabs .tab-selector-4:checked ~ .tabcontent .tabcontent-4{z-index:100;-ms-filter:"progid:DXImageTransform.Microsoft.Alpha(Opacity=100)";filter:alpha(opacity=100);opacity:1;width:100%;height:auto;width:100%;height:auto;padding-top:30px}.tabs .tab-selector-5:checked ~ .tabcontent .tabcontent-5{z-index:100;-ms-filter:"progid:DXImageTransform.Microsoft.Alpha(Opacity=100)";filter:alpha(opacity=100);opacity:1;width:100%;height:auto;width:100%;height:auto;padding-top:30px}.tabs .tab-selector-6:checked ~ .tabcontent .tabcontent-6{z-index:100;-ms-filter:"progid:DXImageTransform.Microsoft.Alpha(Opacity=100)";filter:alpha(opacity=100);opacity:1;width:100%;height:auto;width:100%;height:auto;padding-top:30px}.tabs .tab-selector-7:checked ~ .tabcontent .tabcontent-7{z-index:100;-ms-filter:"progid:DXImageTransform.Microsoft.Alpha(Opacity=100)";filter:alpha(opacity=100);opacity:1;width:100%;height:auto;width:100%;height:auto;padding-top:30px} diff --git a/gremlin-docs/pom.xml b/gremlin-docs/pom.xml new file mode 100644 index 00000000000..c85be734ba2 --- /dev/null +++ b/gremlin-docs/pom.xml @@ -0,0 +1,72 @@ + + + 4.0.0 + + org.apache.tinkerpop + tinkerpop + 3.8.2-SNAPSHOT + + gremlin-docs + Apache TinkerPop :: Gremlin Docs + AsciidoctorJ extension for processing Gremlin code blocks in TinkerPop documentation + + + org.apache.tinkerpop + gremlin-core + ${project.version} + + + org.asciidoctor + asciidoctorj + 2.5.8 + provided + + + org.asciidoctor + asciidoctorj-api + 2.5.8 + provided + + + org.slf4j + slf4j-api + + + + junit + junit + test + + + org.hamcrest + hamcrest + test + + + ch.qos.logback + logback-classic + test + + + + ${basedir}/target + ${project.artifactId}-${project.version} + + diff --git a/gremlin-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/ConsoleExecutor.java b/gremlin-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/ConsoleExecutor.java new file mode 100644 index 00000000000..4c6c9ae6ff4 --- /dev/null +++ b/gremlin-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/ConsoleExecutor.java @@ -0,0 +1,370 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.Closeable; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +/** + * Executes Gremlin code blocks by delegating to a long-running Gremlin Console process + * ({@code bin/gremlin.sh}). Communicates via stdin/stdout using sentinel markers to + * delimit individual statement boundaries. This provides the full console environment + * including correct result formatting, Sugar plugin support, SPARQL, Neo4j, and remote + * connections. + *

+ * The console process is started once and reused across all code blocks in a document, + * maintaining session state (variables, graph bindings) between blocks. + *

+ * Protocol: Each statement is sent individually, followed by a sentinel marker. + * The sentinel is sent twice to handle the console's "Display stack trace? [yN]" error + * prompt, which reads the next stdin line as an answer. If a statement errors, the first + * sentinel is consumed as the "N" answer and the second sentinel produces the expected + * output. This per-statement approach prevents cascading failures where one error could + * consume subsequent code lines or sentinels. + */ +public class ConsoleExecutor implements Closeable { + + private static final Logger log = LoggerFactory.getLogger(ConsoleExecutor.class); + + /** + * Sentinel echoed after each statement to mark the end of output. Chosen to be + * unlikely to appear in normal Gremlin output. + */ + private static final String SENTINEL = "__GREMLIN_DOCS_BLOCK_END__"; + + /** Pattern to strip ANSI escape codes from console output. */ + private static final Pattern ANSI_PATTERN = Pattern.compile("\u001B\\[[0-9;]*[a-zA-Z]"); + + /** Pattern matching the gremlin prompt (with possible ANSI codes already stripped). */ + private static final Pattern PROMPT_PATTERN = Pattern.compile("^gremlin>\\s?"); + + /** Pattern matching continuation prompts like {@code ......1> }. */ + private static final Pattern CONTINUATION_PATTERN = Pattern.compile("^\\.+\\d+>\\s?"); + + private final Process process; + private final BufferedWriter stdin; + private final BufferedReader stdout; + private final Thread stderrDrainer; + + /** + * Creates a new ConsoleExecutor that launches {@code bin/gremlin.sh} from the given + * console home directory. + * + * @param consoleHome path to the unpacked Gremlin Console distribution + */ + public ConsoleExecutor(final String consoleHome) { + this(consoleHome, null); + } + + /** + * Creates a new ConsoleExecutor with an optional {@code HADOOP_GREMLIN_LIBS} setting. + * + * @param consoleHome path to the unpacked Gremlin Console distribution + * @param hadoopGremlinLibs value for the HADOOP_GREMLIN_LIBS environment variable, or null + */ + public ConsoleExecutor(final String consoleHome, final String hadoopGremlinLibs) { + final Path consoleBin = Paths.get(consoleHome, "bin", "gremlin.sh"); + log.info("Starting Gremlin Console from {}", consoleBin); + + try { + final ProcessBuilder pb = new ProcessBuilder(consoleBin.toString()); + pb.directory(Paths.get(consoleHome).toFile()); + pb.environment().put("TERM", "dumb"); + if (hadoopGremlinLibs != null) { + pb.environment().put("HADOOP_GREMLIN_LIBS", hadoopGremlinLibs); + } + + this.process = pb.start(); + this.stdin = new BufferedWriter(new OutputStreamWriter(process.getOutputStream())); + this.stdout = new BufferedReader(new InputStreamReader(process.getInputStream())); + final BufferedReader stderr = new BufferedReader(new InputStreamReader(process.getErrorStream())); + + // drain stderr in background to prevent blocking + this.stderrDrainer = new Thread(() -> { + try { + String line; + while ((line = stderr.readLine()) != null) { + log.debug("console stderr: {}", line); + } + } catch (final Exception ignored) { } + }, "console-stderr-drainer"); + stderrDrainer.setDaemon(true); + stderrDrainer.start(); + + // wait for the console to be ready by consuming the startup banner + sendSentinel(); + consumeUntilSentinel(); + + log.info("Gremlin Console started successfully"); + } catch (final Exception e) { + throw new RuntimeException("Failed to start Gremlin Console from " + consoleHome, e); + } + } + + /** + * Initializes the graph environment for a code block. + * + * @param graph the graph name (modern, classic, crew, sink, grateful) or null/empty for bare TinkerGraph. + * "existing" means reuse the current graph state. + */ + public void initGraph(final String graph) { + if ("existing".equals(graph)) return; + + executeQuietly("if (graph != null && graph instanceof AutoCloseable) graph.close()"); + + if (graph != null && !graph.isEmpty()) { + executeQuietly("graph = TinkerFactory.create" + capitalize(graph) + "()"); + } else { + executeQuietly("graph = TinkerGraph.open()"); + } + executeQuietly("g = graph.traversal()"); + executeQuietly("marko = g.V().has('name', 'marko').tryNext().orElse(null)"); + executeQuietly("f = new File('/tmp/tinkergraph.kryo'); if (f.exists()) f.deleteDir()"); + executeQuietly(":set max-iteration 100"); + } + + /** + * Executes a block of Gremlin code lines and returns the console-formatted output. + * Each statement is sent individually with its own sentinel boundary, so errors + * on one statement cannot consume subsequent statements. + *

+ * Multi-line statements (lines ending with {@code .}, open brackets, etc.) are + * accumulated and sent as a single unit. + */ + public String execute(final List lines) { + final StringBuilder output = new StringBuilder(); + final StringBuilder currentStatement = new StringBuilder(); + final List promptLines = new ArrayList<>(); + + for (final String rawLine : lines) { + final String line = rawLine.replaceAll("(\\s*<\\d+>)+\\s*$", "").trim(); + if (line.isEmpty() || line.startsWith("//")) continue; + + // track prompt display for multi-line statements + if (currentStatement.length() == 0) { + promptLines.add("gremlin> " + line); + } else { + promptLines.add(" " + line); + } + currentStatement.append(line).append("\n"); + + if (isContinuationLine(line, currentStatement.toString())) { + continue; + } + + // complete statement — send it and collect output + final String stmtOutput = executeStatement(currentStatement.toString().trim()); + + // build output: prompt lines followed by result lines + for (final String pl : promptLines) { + output.append(pl).append("\n"); + } + if (!stmtOutput.isEmpty()) { + output.append(stmtOutput); + } + + currentStatement.setLength(0); + promptLines.clear(); + } + + // flush any remaining accumulated statement + if (currentStatement.length() > 0) { + final String stmtOutput = executeStatement(currentStatement.toString().trim()); + for (final String pl : promptLines) { + output.append(pl).append("\n"); + } + if (!stmtOutput.isEmpty()) { + output.append(stmtOutput); + } + } + + return output.toString(); + } + + /** + * Returns the raw Gremlin lines suitable for translation — strips comments, callout markers, + * and joins multi-line continuations into single statements. + */ + public static List extractTranslatableLines(final List lines) { + final List result = new ArrayList<>(); + final StringBuilder current = new StringBuilder(); + + for (String line : lines) { + line = line.replaceAll("(\\s*<\\d+>)+\\s*$", "").trim(); + if (line.isEmpty() || line.startsWith("//") || line.startsWith(":")) continue; + + current.append(line).append("\n"); + + if (!isContinuationLine(line, current.toString())) { + result.add(current.toString().trim()); + current.setLength(0); + } + } + + if (current.length() > 0) { + result.add(current.toString().trim()); + } + + return result; + } + + /** + * Determines if the current accumulated statement is incomplete and needs more lines. + */ + static boolean isContinuationLine(final String trimmedLine, final String accumulated) { + if (trimmedLine.endsWith(".") || trimmedLine.endsWith("{") || trimmedLine.endsWith(",") || + trimmedLine.endsWith("(") || trimmedLine.endsWith("\\")) { + return true; + } + return countChar(accumulated, '(') > countChar(accumulated, ')') || + countChar(accumulated, '[') > countChar(accumulated, ']') || + countChar(accumulated, '{') > countChar(accumulated, '}'); + } + + @Override + public void close() { + try { + sendLine(":exit"); + stdin.flush(); + stdin.close(); + } catch (final Exception ignored) { } + + try { + process.waitFor(10, TimeUnit.SECONDS); + } catch (final InterruptedException ignored) { } + + if (process.isAlive()) { + process.destroyForcibly(); + } + } + + /** + * Sends a single statement to the console, followed by a double sentinel, and reads + * back only the result lines (everything between the prompt echo and the sentinel). + * Returns the result lines (e.g. {@code ==>6\n}) or empty string if no results. + */ + private String executeStatement(final String statement) { + final StringBuilder result = new StringBuilder(); + try { + sendLine(statement); + sendSentinel(); + + String line; + while ((line = stdout.readLine()) != null) { + line = stripAnsi(line); + if (line.contains(SENTINEL)) break; + + // skip prompt lines — we build our own prompt display + if (PROMPT_PATTERN.matcher(line).find()) continue; + if (CONTINUATION_PATTERN.matcher(line).find()) continue; + + // capture result lines + if (line.startsWith("==>")) { + result.append(line).append("\n"); + } + // other non-prompt output (e.g. println from scripts) is included + else if (!line.isEmpty()) { + result.append(line).append("\n"); + } + } + } catch (final Exception e) { + log.error("Error executing statement: {}", statement, e); + } + return result.toString(); + } + + /** + * Sends a statement and discards all output until the sentinel. + */ + private void executeQuietly(final String statement) { + try { + sendLine(statement); + sendSentinel(); + consumeUntilSentinel(); + } catch (final Exception e) { + log.warn("Error during quiet execution: {}", statement, e); + } + } + + /** + * Sends the sentinel marker twice. The double-send handles the case where the console + * encounters an error and prompts "Display stack trace? [yN]" — that prompt reads the + * next stdin line as the answer, consuming the first sentinel. The second sentinel + * ensures we still see it in stdout and don't hang. The sentinel text doesn't start + * with "y"/"Y", so no stack trace is printed. + */ + private void sendSentinel() { + sendLine("'" + SENTINEL + "'"); + sendLine("'" + SENTINEL + "'"); + } + + private void sendLine(final String line) { + try { + stdin.write(line); + stdin.newLine(); + stdin.flush(); + } catch (final Exception e) { + throw new RuntimeException("Failed to write to console stdin", e); + } + } + + /** + * Reads and discards stdout lines until the sentinel marker is found. + */ + private void consumeUntilSentinel() { + try { + String line; + while ((line = stdout.readLine()) != null) { + line = stripAnsi(line); + if (line.contains(SENTINEL)) return; + } + } catch (final Exception e) { + log.warn("Error consuming console output", e); + } + } + + private static String stripAnsi(final String s) { + return ANSI_PATTERN.matcher(s).replaceAll(""); + } + + private static String capitalize(final String s) { + if (s == null || s.isEmpty()) return s; + return Character.toUpperCase(s.charAt(0)) + s.substring(1); + } + + private static int countChar(final String s, final char c) { + int count = 0; + for (int i = 0; i < s.length(); i++) { + if (s.charAt(i) == c) count++; + } + return count; + } +} diff --git a/docs/preprocessor/awk/progressbar.groovy.template b/gremlin-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinDocsExtension.java similarity index 58% rename from docs/preprocessor/awk/progressbar.groovy.template rename to gremlin-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinDocsExtension.java index 7fcafea8e9c..5261b07700e 100644 --- a/docs/preprocessor/awk/progressbar.groovy.template +++ b/gremlin-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinDocsExtension.java @@ -16,20 +16,19 @@ * specific language governing permissions and limitations * under the License. */ +package org.apache.tinkerpop.gremlin.docs; + +import org.asciidoctor.Asciidoctor; +import org.asciidoctor.jruby.extension.spi.ExtensionRegistry; /** - * @author Daniel Kuppitz (http://gremlin.guru) + * SPI entry point that registers the {@link GremlinTreeProcessor} with AsciidoctorJ. + * Discovered automatically via {@code META-INF/services/org.asciidoctor.jruby.extension.spi.ExtensionRegistry}. */ -pb = { def progress -> - def barLength = 100 - def ratio = barLength / 100 - def builder = new StringBuilder() - def percent = (int) ((progress / TOTAL_LINES) * 100) - def progressLength = (int) ((progress / TOTAL_LINES) * (100 * ratio)) - builder.append('=' * progressLength) - if (progressLength < barLength) { - builder.append('>') - builder.append(' ' * (barLength - progressLength - 1)) - } - System.err.print(String.format("\r progress: [%s] %s", builder, "${percent}%")) +public class GremlinDocsExtension implements ExtensionRegistry { + + @Override + public void register(final Asciidoctor asciidoctor) { + asciidoctor.javaExtensionRegistry().treeprocessor(GremlinTreeProcessor.class); + } } diff --git a/gremlin-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinTreeProcessor.java b/gremlin-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinTreeProcessor.java new file mode 100644 index 00000000000..fef5a12f11b --- /dev/null +++ b/gremlin-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/GremlinTreeProcessor.java @@ -0,0 +1,363 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.asciidoctor.ast.Block; +import org.asciidoctor.ast.Document; +import org.asciidoctor.ast.StructuralNode; +import org.asciidoctor.extension.Treeprocessor; +import org.apache.tinkerpop.gremlin.language.translator.Translator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * AsciidoctorJ {@link Treeprocessor} that processes {@code [gremlin-groovy,modern]} code blocks + * in TinkerPop documentation. For each such block, it: + *

    + *
  1. Executes the Gremlin code via a {@link ConsoleExecutor} (real Gremlin Console process) and captures console output
  2. + *
  3. Translates the canonical Gremlin to all language variants via {@link VariantTranslator}
  4. + *
  5. Wraps the console output and translations in a tabbed UI with proper AST listing blocks + * so Asciidoctor applies syntax highlighting via CodeRay
  6. + *
+ */ +public class GremlinTreeProcessor extends Treeprocessor { + + private static final Logger log = LoggerFactory.getLogger(GremlinTreeProcessor.class); + private static final Pattern GREMLIN_STYLE = Pattern.compile("gremlin-(\\w+)"); + private static final AtomicLong counter = new AtomicLong(System.currentTimeMillis()); + + @Override + public Document process(final Document document) { + final boolean dryRun = document.hasAttribute("gremlin-docs-dryrun"); + final String consoleHome = document.hasAttribute("gremlin-docs-console-home") + ? document.getAttribute("gremlin-docs-console-home").toString() + : null; + final String hadoopLibs = document.hasAttribute("gremlin-docs-hadoop-libs") + ? document.getAttribute("gremlin-docs-hadoop-libs").toString() + : null; + + if (dryRun || consoleHome == null) { + processNode(document, null, true); + } else { + try (final ConsoleExecutor executor = new ConsoleExecutor(consoleHome, hadoopLibs)) { + processNode(document, executor, false); + } + } + + return document; + } + + private void processNode(final StructuralNode node, final ConsoleExecutor executor, final boolean dryRun) { + final List blocks = node.getBlocks(); + if (blocks == null || blocks.isEmpty()) return; + + for (int i = 0; i < blocks.size(); i++) { + final StructuralNode child = blocks.get(i); + + if (child instanceof Block && isGremlinBlock((Block) child)) { + i = processGremlinBlock(node, i, (Block) child, executor, dryRun); + } else if (child instanceof Block && isTabStartBlock((Block) child)) { + i = processStandaloneTabGroup(node, i); + } else { + processNode(child, executor, dryRun); + } + } + } + + /** + * Replaces a gremlin block with a sequence of AST nodes that form a tabbed view: + * passthrough HTML for the tab structure interleaved with real listing blocks that + * Asciidoctor will syntax-highlight. + */ + private int processGremlinBlock(final StructuralNode parent, final int index, + final Block block, final ConsoleExecutor executor, + final boolean dryRun) { + final Matcher m = GREMLIN_STYLE.matcher(block.getStyle()); + if (!m.matches()) return index; + + final String lang = m.group(1); + final String graph = getGraphAttribute(block); + final List lines = block.getLines(); + + log.info("Processing [gremlin-{},{}] block ({} lines)", lang, + graph != null ? graph : "", lines.size()); + + // execute the gremlin code + String consoleOutput; + if (dryRun || executor == null) { + consoleOutput = formatDryRun(lines); + } else { + try { + executor.initGraph(graph); + consoleOutput = executor.execute(lines); + } catch (final Exception e) { + log.error("Failed to execute gremlin block", e); + consoleOutput = formatDryRun(lines); + } + } + + // collect tab entries: label + language + code content + final List tabs = new ArrayList<>(); + tabs.add(new TabEntry("console", "groovy", consoleOutput)); + + // translate to language variants (available on 4.0+ with ANTLR-based translator) + final List translatableLines = ConsoleExecutor.extractTranslatableLines(lines); + if (!translatableLines.isEmpty()) { + final Map translations = VariantTranslator.translateBlock(translatableLines); + for (final Map.Entry entry : translations.entrySet()) { + tabs.add(new TabEntry( + VariantTranslator.getDisplayName(entry.getKey()), + VariantTranslator.getSourceLanguage(entry.getKey()), + entry.getValue())); + } + } + + // consume any following [source,LANG,tab] blocks + final List siblings = parent.getBlocks(); + int nextIndex = index + 1; + while (nextIndex < siblings.size()) { + final StructuralNode next = siblings.get(nextIndex); + if (next instanceof Block && isManualTabBlock((Block) next)) { + final Block tabBlock = (Block) next; + final String tabLang = getSourceLanguage(tabBlock); + tabs.add(new TabEntry( + tabLang != null ? tabLang : "code", + tabLang, + String.join("\n", tabBlock.getLines()))); + nextIndex++; + } else { + break; + } + } + + // build the replacement AST nodes + final List replacements = buildTabbedBlocks(parent, tabs); + + // replace original block and consumed tab blocks with the new sequence + // remove consumed blocks first (backwards to preserve indices) + for (int j = nextIndex - 1; j > index; j--) { + siblings.remove(j); + } + // remove the original gremlin block + siblings.remove(index); + // insert replacements at the same position + siblings.addAll(index, replacements); + + // return last index of inserted blocks so the loop continues after them + return index + replacements.size() - 1; + } + + /** + * Builds a sequence of AST blocks: passthrough HTML for tab structure interleaved + * with real listing blocks for syntax-highlighted code. + */ + private List buildTabbedBlocks(final StructuralNode parent, final List tabs) { + final List nodes = new ArrayList<>(); + + final long id = counter.incrementAndGet(); + final int numTabs = tabs.size(); + + // opening HTML: section + radio buttons + labels + first tab content div open + final StringBuilder openHtml = new StringBuilder(); + openHtml.append("
\n"); + for (int i = 0; i < numTabs; i++) { + final int tabNum = i + 1; + final String checked = (i == 0) ? " checked=\"checked\"" : ""; + openHtml.append(" \n"); + openHtml.append(" \n"); + } + openHtml.append("
\n
\n"); + nodes.add(createBlock(parent, "pass", openHtml.toString())); + + // first tab content (listing block) + nodes.add(createListingBlock(parent, tabs.get(0).language, tabs.get(0).content)); + + // remaining tabs: close previous div, open next div, listing block + for (int i = 1; i < numTabs; i++) { + final int tabNum = i + 1; + final String divHtml = "
\n
\n" + + "
\n
\n"; + nodes.add(createBlock(parent, "pass", divHtml)); + nodes.add(createListingBlock(parent, tabs.get(i).language, tabs.get(i).content)); + } + + // closing HTML + nodes.add(createBlock(parent, "pass", "
\n
\n
")); + + return nodes; + } + + /** + * Creates a proper Asciidoctor source listing block by parsing AsciiDoc markup. + * This ensures CodeRay syntax highlighting is applied, since the block goes through + * Asciidoctor's normal parsing pipeline. + */ + private Block createListingBlock(final StructuralNode parent, final String language, final String content) { + final List lines = new ArrayList<>(); + lines.add("[source," + language + "]"); + lines.add("----"); + for (final String line : content.split("\n", -1)) { + lines.add(line); + } + lines.add("----"); + final int sizeBefore = parent.getBlocks().size(); + parseContent(parent, lines); + final List blocks = parent.getBlocks(); + if (blocks.size() > sizeBefore) { + return (Block) blocks.remove(blocks.size() - 1); + } + // fallback if parseContent produced nothing + return (Block) createBlock(parent, "listing", content); + } + + private boolean isGremlinBlock(final Block block) { + final String style = block.getStyle(); + return style != null && GREMLIN_STYLE.matcher(style).matches(); + } + + /** + * Checks if a block starts a standalone tab group: {@code [source,LANG,tab]}. + */ + private boolean isTabStartBlock(final Block block) { + if (!"source".equals(block.getStyle())) return false; + final Map attrs = block.getAttributes(); + // "tab" can appear as attribute "2" or "3" depending on how asciidoctor parses positions + return "tab".equals(attrs.get("2")) || "tab".equals(attrs.get("3")); + } + + /** + * Checks if a block is a continuation of a tab group: a {@code [source,LANG]} block + * whose language hasn't already been seen in the group. + */ + private boolean isTabContinuationBlock(final Block block, final java.util.Set seenLanguages) { + if (!"source".equals(block.getStyle())) return false; + final String lang = getSourceLanguage(block); + return lang != null && !seenLanguages.contains(lang); + } + + private boolean isManualTabBlock(final Block block) { + if (!"source".equals(block.getStyle())) return false; + final Map attrs = block.getAttributes(); + return "tab".equals(attrs.get("2")) || "tab".equals(attrs.get("3")); + } + + /** + * Processes a standalone tab group starting with {@code [source,LANG,tab]} and collecting + * all consecutive {@code [source,LANG]} blocks into a tabbed view. + */ + private int processStandaloneTabGroup(final StructuralNode parent, final int index) { + final List siblings = parent.getBlocks(); + final List tabs = new ArrayList<>(); + + // collect the first block and all consecutive source blocks + final Set seenLanguages = new HashSet<>(); + int nextIndex = index; + while (nextIndex < siblings.size()) { + final StructuralNode node = siblings.get(nextIndex); + if (!(node instanceof Block)) break; + final Block block = (Block) node; + + if (nextIndex == index) { + // first block must be a tab-start block + if (!isTabStartBlock(block)) break; + } else { + // subsequent blocks must be source blocks with a unique language + if (!isTabContinuationBlock(block, seenLanguages)) break; + } + + final String lang = getSourceLanguage(block); + final String label = lang != null ? lang : "code"; + if (lang != null) seenLanguages.add(lang); + tabs.add(new TabEntry(label, lang, String.join("\n", block.getLines()))); + nextIndex++; + } + + if (tabs.size() <= 1) return index; // not enough blocks for tabs + + log.info("Processing standalone tab group ({} tabs)", tabs.size()); + + final List replacements = buildTabbedBlocks(parent, tabs); + + // remove original blocks (backwards) + for (int j = nextIndex - 1; j >= index; j--) { + siblings.remove(j); + } + siblings.addAll(index, replacements); + + return index + replacements.size() - 1; + } + + private String getGraphAttribute(final Block block) { + final Map attrs = block.getAttributes(); + Object attr = attrs.get("2"); + if (attr == null) attr = attrs.get(2); + if (attr == null || "false".equals(attr.toString()) || attr.toString().isEmpty()) return null; + return attr.toString(); + } + + private String getSourceLanguage(final Block block) { + // For [source,LANG], asciidoctor may store the language in "language" attr, + // attribute "1" (which may contain "source"), or attribute "2". + final Object langAttr = block.getAttribute("language"); + if (langAttr != null) return langAttr.toString(); + final Map attrs = block.getAttributes(); + // attribute "1" is often the style name itself; "2" has the language + final Object attr2 = attrs.get("2"); + if (attr2 != null && !"tab".equals(attr2.toString()) && !"false".equals(attr2.toString())) { + return attr2.toString(); + } + final Object attr1 = attrs.get("1"); + if (attr1 != null && !"source".equals(attr1.toString())) return attr1.toString(); + return null; + } + + private static String formatDryRun(final List lines) { + final StringBuilder sb = new StringBuilder(); + for (final String line : lines) { + sb.append("gremlin> ").append(line).append("\n"); + } + return sb.toString(); + } + + private static class TabEntry { + final String label; + final String language; + final String content; + + TabEntry(final String label, final String language, final String content) { + this.label = label; + this.language = language; + this.content = content; + } + } +} diff --git a/gremlin-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/VariantTranslator.java b/gremlin-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/VariantTranslator.java new file mode 100644 index 00000000000..ef5bd8b64a6 --- /dev/null +++ b/gremlin-docs/src/main/java/org/apache/tinkerpop/gremlin/docs/VariantTranslator.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.apache.tinkerpop.gremlin.language.translator.GremlinTranslator; +import org.apache.tinkerpop.gremlin.language.translator.Translation; +import org.apache.tinkerpop.gremlin.language.translator.Translator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Translates canonical Gremlin into all supported language variants using {@link GremlinTranslator}. + */ +public class VariantTranslator { + + private static final Logger log = LoggerFactory.getLogger(VariantTranslator.class); + + /** + * The language variants to generate, in display order. Excludes CANONICAL, ANONYMIZED, GROOVY + * (which is essentially the same as the console output), and LANGUAGE (deprecated). + */ + static final List VARIANT_LANGUAGES = Collections.unmodifiableList(Arrays.asList( + Translator.JAVA, + Translator.PYTHON, + Translator.JAVASCRIPT, + Translator.DOTNET, + Translator.GO + )); + + /** + * Display names for tab labels. + */ + private static final Map DISPLAY_NAMES = new LinkedHashMap<>(); + static { + DISPLAY_NAMES.put(Translator.JAVA, "java"); + DISPLAY_NAMES.put(Translator.PYTHON, "python"); + DISPLAY_NAMES.put(Translator.JAVASCRIPT, "javascript"); + DISPLAY_NAMES.put(Translator.DOTNET, "c#"); + DISPLAY_NAMES.put(Translator.GO, "go"); + } + + /** + * Asciidoc source language identifiers for syntax highlighting. + */ + private static final Map SOURCE_LANGUAGES = new LinkedHashMap<>(); + static { + SOURCE_LANGUAGES.put(Translator.JAVA, "java"); + SOURCE_LANGUAGES.put(Translator.PYTHON, "python"); + SOURCE_LANGUAGES.put(Translator.JAVASCRIPT, "javascript"); + SOURCE_LANGUAGES.put(Translator.DOTNET, "csharp"); + SOURCE_LANGUAGES.put(Translator.GO, "go"); + } + + public static String getDisplayName(final Translator translator) { + return DISPLAY_NAMES.getOrDefault(translator, translator.getName().toLowerCase()); + } + + public static String getSourceLanguage(final Translator translator) { + return SOURCE_LANGUAGES.getOrDefault(translator, translator.getName().toLowerCase()); + } + + /** + * Translates a single Gremlin statement to all variant languages. Returns a map from + * {@link Translator} to the translated code string. Statements that fail to parse + * (e.g. those containing lambdas or non-standard Groovy) are skipped with a warning. + */ + public static Map translateStatement(final String gremlin) { + final Map results = new LinkedHashMap<>(); + for (final Translator lang : VARIANT_LANGUAGES) { + try { + final Translation t = GremlinTranslator.translate(gremlin, "g", lang); + results.put(lang, t.getTranslated()); + } catch (final Exception e) { + log.debug("Cannot translate to {}: {} — {}", lang.getName(), gremlin, e.getMessage()); + } + } + return results; + } + + /** + * Translates multiple Gremlin statements and joins them with newlines per language. + * If any statement fails to translate for a given language, that language is omitted entirely. + */ + public static Map translateBlock(final List statements) { + final Map results = new LinkedHashMap<>(); + + for (final Translator lang : VARIANT_LANGUAGES) { + final StringBuilder sb = new StringBuilder(); + boolean allTranslated = true; + + for (final String stmt : statements) { + try { + final Translation t = GremlinTranslator.translate(stmt, "g", lang); + if (sb.length() > 0) sb.append("\n"); + sb.append(t.getTranslated()); + } catch (final Exception e) { + log.debug("Cannot translate to {}: {} — {}", lang.getName(), stmt, e.getMessage()); + allTranslated = false; + break; + } + } + + if (allTranslated) { + results.put(lang, sb.toString()); + } + } + + return results; + } +} diff --git a/gremlin-docs/src/main/resources/META-INF/services/org.asciidoctor.jruby.extension.spi.ExtensionRegistry b/gremlin-docs/src/main/resources/META-INF/services/org.asciidoctor.jruby.extension.spi.ExtensionRegistry new file mode 100644 index 00000000000..6a81ac1f60e --- /dev/null +++ b/gremlin-docs/src/main/resources/META-INF/services/org.asciidoctor.jruby.extension.spi.ExtensionRegistry @@ -0,0 +1 @@ +org.apache.tinkerpop.gremlin.docs.GremlinDocsExtension diff --git a/gremlin-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinDocsTest.java b/gremlin-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinDocsTest.java new file mode 100644 index 00000000000..30d6b6977eb --- /dev/null +++ b/gremlin-docs/src/test/java/org/apache/tinkerpop/gremlin/docs/GremlinDocsTest.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.docs; + +import org.apache.tinkerpop.gremlin.language.translator.Translator; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class GremlinDocsTest { + + @Test + public void shouldExtractTranslatableLines() { + final List lines = Arrays.asList( + "g.V().has('name','marko'). <1>", + " out('knows').values('name') <2>", + "// this is a comment", + "g.V().count()" + ); + final List result = ConsoleExecutor.extractTranslatableLines(lines); + assertEquals(2, result.size()); + assertEquals("g.V().has('name','marko').\nout('knows').values('name')", result.get(0)); + assertEquals("g.V().count()", result.get(1)); + } + + @Test + public void shouldDetectContinuationLines() { + assertThat(ConsoleExecutor.isContinuationLine("g.V().", "g.V()."), is(true)); + assertThat(ConsoleExecutor.isContinuationLine("has('name','marko')", "g.V().\nhas('name','marko')"), is(false)); + assertThat(ConsoleExecutor.isContinuationLine("map{", "map{"), is(true)); + assertThat(ConsoleExecutor.isContinuationLine("[1,", "[1,"), is(true)); + } + + @Test + public void shouldSkipConsoleCommandsInExtraction() { + final List lines = Arrays.asList( + ":remote connect tinkerpop.server conf/remote.yaml", + ":> g.V().count()", + "g.V().count()" + ); + final List result = ConsoleExecutor.extractTranslatableLines(lines); + assertEquals(1, result.size()); + assertEquals("g.V().count()", result.get(0)); + } + + @Test + public void shouldTranslateToVariants() { + final Map translations = VariantTranslator.translateStatement( + "g.V().has('name','marko').out('knows').values('name')"); + + assertFalse(translations.isEmpty()); + assertTrue(translations.containsKey(Translator.PYTHON)); + assertTrue(translations.containsKey(Translator.JAVA)); + assertTrue(translations.containsKey(Translator.JAVASCRIPT)); + assertTrue(translations.containsKey(Translator.DOTNET)); + assertTrue(translations.containsKey(Translator.GO)); + + assertTrue(translations.get(Translator.PYTHON).contains("has(")); + assertTrue(translations.get(Translator.PYTHON).contains("out(")); + } + + @Test + public void shouldTranslateBlock() { + final List statements = Arrays.asList( + "g.V().has('name','marko').out('knows').values('name')", + "g.V().count()" + ); + final Map translations = VariantTranslator.translateBlock(statements); + + assertFalse(translations.isEmpty()); + for (final String code : translations.values()) { + assertTrue(code.contains("\n")); + } + } + + @Test + public void shouldSkipUntranslatableStatements() { + final Map translations = VariantTranslator.translateStatement( + "g.V().filter{it.get().label() == 'person'}"); + assertNotNull(translations); + } +} diff --git a/pom.xml b/pom.xml index aaef1b2fdea..3c2f0fc89a6 100644 --- a/pom.xml +++ b/pom.xml @@ -956,14 +956,12 @@ limitations under the License. - - ${project.basedir}/target/postprocess-asciidoc + + ${project.basedir}/docs/src - + ${project.basedir}/target/doc-source @@ -1035,6 +1033,29 @@ limitations under the License. org.asciidoctor asciidoctor-maven-plugin false + + + org.apache.tinkerpop + gremlin-docs + ${project.version} + + + + org.apache.commons + commons-text + 1.15.0 + + + + org.yaml + snakeyaml + 1.33 + + home @@ -1065,7 +1086,7 @@ limitations under the License. UTF-8 ${asciidoctor.style.dir} tinkerpop.css - coderay + rouge ${project.basedir} shared ${project.basedir}/docs/src @@ -1092,7 +1113,7 @@ limitations under the License. left ${asciidoctor.style.dir} tinkerpop.css - coderay + rouge ${project.basedir} shared ${project.basedir}/docs/src @@ -1119,7 +1140,7 @@ limitations under the License. left ${asciidoctor.style.dir} tinkerpop.css - coderay + rouge ${project.basedir} shared ${project.basedir}/docs/src @@ -1146,7 +1167,7 @@ limitations under the License. left ${asciidoctor.style.dir} tinkerpop.css - coderay + rouge ${project.basedir} shared ${project.basedir}/docs/src @@ -1173,7 +1194,7 @@ limitations under the License. left ${asciidoctor.style.dir} tinkerpop.css - coderay + rouge ${project.basedir} shared ${project.basedir}/docs/src @@ -1200,7 +1221,7 @@ limitations under the License. left ${asciidoctor.style.dir} tinkerpop.css - coderay + rouge ${project.basedir} shared ${project.basedir}/docs/src @@ -1227,7 +1248,7 @@ limitations under the License. left ${asciidoctor.style.dir} tinkerpop.css - coderay + rouge ${project.basedir} shared ${project.basedir}/docs/src @@ -1254,7 +1275,7 @@ limitations under the License. left ${asciidoctor.style.dir} tinkerpop.css - coderay + rouge ${project.basedir} shared ${project.basedir}/docs/src @@ -1279,7 +1300,7 @@ limitations under the License. UTF-8 ${asciidoctor.style.dir} tinkerpop.css - coderay + rouge ${project.basedir} shared ${project.basedir}/docs/src @@ -1305,7 +1326,7 @@ limitations under the License. UTF-8 ${asciidoctor.style.dir} tinkerpop.css - coderay + rouge ${project.basedir} shared ${project.basedir}/docs/src @@ -1331,7 +1352,7 @@ limitations under the License. UTF-8 ${asciidoctor.style.dir} tinkerpop.css - coderay + rouge ${project.basedir} shared ${project.basedir}/docs/src @@ -1356,7 +1377,7 @@ limitations under the License. UTF-8 ${asciidoctor.style.dir} tinkerpop.css - coderay + rouge ${project.basedir} shared ${project.basedir}/docs/src