#!/usr/bin/env bash

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"

PYTHON_BIN="${PYTHON_BIN:-python3}"

DEFAULT_VAULT_ROOT="/home/ermis/products/redstack/redstack-vault"

DEFAULT_SYSTEM_PROMPT="${PROJECT_ROOT}/prompts/system-prompt.example.txt"

DEFAULT_OUTPUT_ROOT="${PROJECT_ROOT}/data/redstack-vault"

DEFAULT_VENV_PATH="${PROJECT_ROOT}/.venv"

MODE="dataset"

VAULT_ROOT="${DEFAULT_VAULT_ROOT}"

SYSTEM_PROMPT_PATH="${DEFAULT_SYSTEM_PROMPT}"

OUTPUT_ROOT="${DEFAULT_OUTPUT_ROOT}"

VENV_PATH="${DEFAULT_VENV_PATH}"

RUN_ID="${RUN_ID:-$(date -u +"%Y%m%dT%H%M%SZ")}"

MANIFEST_PATH=""

MAX_EXAMPLES=""

SKIP_INSTALL=0

REFRESH_VENV=0

usage() {

cat <<'EOF'

Usage:

bin/stackbridge-redstack-dataset [options]

Options:

--mode dataset|manifest Build a dataset or just emit a source manifest. Default: dataset

--vault-root PATH Source vault root. Default: /home/ermis/products/redstack/redstack-vault

--output-root PATH Base output directory. Default: data/redstack-vault

--run-id ID Subdirectory name under output-root. Default: current UTC timestamp

--system-prompt PATH System prompt file for dataset builds

--manifest-path PATH Reuse an existing manifest instead of generating one

--max-examples N Limit dataset generation for testing

--python BIN Base Python interpreter to use for venv creation. Default: python3

--venv PATH Virtualenv path. Default: .venv

--skip-install Do not install or refresh pip dependencies

--refresh-venv Reinstall requirements into the venv even if already present

--help Show this help

Examples:

bin/stackbridge-redstack-dataset

bin/stackbridge-redstack-dataset --mode manifest

bin/stackbridge-redstack-dataset --vault-root /some/other/vault --run-id test-run

EOF

}

while [[ $# -gt 0 ]]; do

case "$1" in

--mode)

MODE="$2"

shift 2

;;

--vault-root)

VAULT_ROOT="$2"

shift 2

;;

--output-root)

OUTPUT_ROOT="$2"

shift 2

;;

--run-id)

RUN_ID="$2"

shift 2

;;

--system-prompt)

SYSTEM_PROMPT_PATH="$2"

shift 2

;;

--manifest-path)

MANIFEST_PATH="$2"

shift 2

;;

--max-examples)

MAX_EXAMPLES="$2"

shift 2

;;

--python)

PYTHON_BIN="$2"

shift 2

;;

--venv)

VENV_PATH="$2"

shift 2

;;

--skip-install)

SKIP_INSTALL=1

shift

;;

--refresh-venv)

REFRESH_VENV=1

shift

;;

--help|-h)

usage

exit 0

;;

*)

echo "Unknown argument: $1" >&2

usage >&2

exit 1

;;

esac

done

case "${MODE}" in

dataset|manifest)

;;

*)

echo "Unsupported mode: ${MODE}. Use dataset or manifest." >&2

exit 1

;;

esac

if ! command -v "${PYTHON_BIN}" >/dev/null 2>&1; then

echo "Python interpreter not found: ${PYTHON_BIN}" >&2

exit 1

fi

if [[ ! -x "${VENV_PATH}/bin/python" ]]; then

echo "Creating virtualenv at ${VENV_PATH}"

"${PYTHON_BIN}" -m venv "${VENV_PATH}"

REFRESH_VENV=1

fi

VENV_PYTHON="${VENV_PATH}/bin/python"

VENV_PIP="${VENV_PATH}/bin/pip"

REQUIREMENTS_FILE="${PROJECT_ROOT}/requirements.txt"

REQUIREMENTS_STAMP="${VENV_PATH}/.stackbridge_requirements_installed"

if [[ "${SKIP_INSTALL}" -eq 0 ]]; then

if [[ "${REFRESH_VENV}" -eq 1 || ! -f "${REQUIREMENTS_STAMP}" || "${REQUIREMENTS_FILE}" -nt "${REQUIREMENTS_STAMP}" ]]; then

echo "Installing Python dependencies into ${VENV_PATH}"

"${VENV_PIP}" install --upgrade pip

"${VENV_PIP}" install -r "${REQUIREMENTS_FILE}"

touch "${REQUIREMENTS_STAMP}"

fi

fi

if [[ ! -d "${VAULT_ROOT}" ]]; then

echo "Vault root does not exist: ${VAULT_ROOT}" >&2

exit 1

fi

RUN_OUTPUT_DIR="${OUTPUT_ROOT}/${RUN_ID}"

if [[ -z "${MANIFEST_PATH}" ]]; then

MANIFEST_PATH="${RUN_OUTPUT_DIR}/source_manifest.jsonl"

fi

DATASET_PATH="${RUN_OUTPUT_DIR}/train.jsonl"

METADATA_PATH="${RUN_OUTPUT_DIR}/train.meta.jsonl"

CONFIG_PATH="${RUN_OUTPUT_DIR}/redstack-vault.runtime.json"

mkdir -p "${RUN_OUTPUT_DIR}"

if [[ "${MODE}" == "manifest" ]]; then

echo "Writing manifest to ${MANIFEST_PATH}"

exec "${VENV_PYTHON}" "${PROJECT_ROOT}/bin/stackbridge-ingest-tree" \

--source-root "${VAULT_ROOT}" \

--output "${MANIFEST_PATH}" \

--include-glob "**/*.md" \

--include-glob "**/*.txt" \

--include-glob "**/*.json"

fi

if [[ ! -f "${MANIFEST_PATH}" ]]; then

echo "Manifest not found, generating one at ${MANIFEST_PATH}"

"${VENV_PYTHON}" "${PROJECT_ROOT}/bin/stackbridge-ingest-tree" \

--source-root "${VAULT_ROOT}" \

--output "${MANIFEST_PATH}" \

--include-glob "**/*.md" \

--include-glob "**/*.txt" \

--include-glob "**/*.json"

fi

if [[ ! -f "${SYSTEM_PROMPT_PATH}" ]]; then

echo "System prompt file does not exist: ${SYSTEM_PROMPT_PATH}" >&2

exit 1

fi

cat > "${CONFIG_PATH}" <<EOF

{

"adapter": "stackbridge.adapters.redstack_vault:RedstackVaultAdapter",

"adapter_config": {

"vault_root": "${VAULT_ROOT}",

"manifest_path": "${MANIFEST_PATH}",

"include_note_types": [

"procedures",

"attack-chains"

],

"reference_note_types": [

"commands",

"tools",

"codes"

],

"require_validated": true,

"require_verified": false,

"skip_truncated": true,

"max_reference_notes_per_example": 3,

"include_command_references": true,

"include_tool_references": true,

"include_code_references": false,

"include_globs": [

"**/*.md",

"**/*.txt",

"**/*.json"

],

"exclude_globs": [

"**/.git/**",

"**/.obsidian/**",

"**/_assets/**",

"**/assets/images/**",

"**/assets/files/originals/**",

"**/node_modules/**",

"**/.venv/**",

"**/venv/**",

"**/dist/**",

"**/build/**"

]$(if [[ -n "${MAX_EXAMPLES}" ]]; then printf ',\n "max_examples": %s' "${MAX_EXAMPLES}"; fi)

},

"sanitizers": [

{

"callable": "stackbridge.sanitizers.builtin:ObsidianLinkSanitizer",

"config": {

"replace_hyphens": true

}

},

{

"callable": "stackbridge.sanitizers.builtin:SecurityRedactionSanitizer",

"config": {

"redact_emails": true,

"redact_jwts": true,

"redact_bearer_tokens": true,

"redact_aws_access_keys": true,

"redact_assignment_values": true

}

},

{

"callable": "stackbridge.sanitizers.builtin:WhitespaceNormalizerSanitizer"

}

],

"default_system_prompt_path": "${SYSTEM_PROMPT_PATH}",

"output_dataset_path": "${DATASET_PATH}",

"output_metadata_path": "${METADATA_PATH}"

}

EOF

echo "Writing runtime config to ${CONFIG_PATH}"

echo "Building dataset from ${VAULT_ROOT}"

"${VENV_PYTHON}" "${PROJECT_ROOT}/bin/stackbridge-build-dataset" --config "${CONFIG_PATH}"

echo

echo "Done."

echo "Manifest: ${MANIFEST_PATH}"

echo "Dataset : ${DATASET_PATH}"

echo "Metadata: ${METADATA_PATH}"

echo "Config : ${CONFIG_PATH}"

echo "Venv : ${VENV_PATH}"