Local Testing Procedure

Making a fresh dirctory

Made the file make_local_test_env.sh in ox_lvra so I don’t touch the data and code directories that I know are working like on the server.

#!/usr/bin/env bash

years_arr=(2026
    2027
    2028
    2029
    2030
    2031
    2032
    2033
    2034
    2035
)

today=$(date +"%Y%m%d")
today_year=$(date +"%Y")

mkdir -p "test_$today"

cd "test_$today"

mkdir -p data/lvra
mkdir -p code

cd data/lvra

work_dir=$(pwd)

mkdir -p JSON
mkdir -p csv
mkdir -p logs
mkdir -p db


for dir in JSON csv logs; do
    for year in "${years_arr[@]}"; do
        mkdir -p "$work_dir/$dir/$year"
        done

    mkdir mkdir -p "$work_dir/$dir/$today_year/$today"


done

cd "$work_dir/db"

cat > log_schema.sql <<'SQL'
CREATE TABLE IF NOT EXISTS feature_making (
    stem TEXT PRIMARY KEY,
    timestamp TEXT NOT NULL DEFAULT current_timestamp,
    r0b INTEGER
);

CREATE TABLE IF NOT EXISTS annotating (
    stem TEXT PRIMARY KEY,
    timestamp TEXT NOT NULL DEFAULT current_timestamp,
    r0b INTEGER
);

CREATE TABLE IF NOT EXISTS diaobjid_stems (
    diaObjectId INTEGER PRIMARY KEY,
    stem TEXT NOT NULL,
    timestamp TEXT NOT NULL DEFAULT current_timestamp
);

CREATE TABLE IF NOT EXISTS provenance (
    ID INTEGER PRIMARY KEY,
    diaObjectId INTEGER,
    diaSourceId INTEGER,
    stem TEXT,
    score REAL,
    model_name TEXT,
    model_version TEXT,
    timestamp TEXT NOT NULL DEFAULT current_timestamp
);
SQL

sqlite3 log.db < log_schema.sql

## DATA
TEST_JSON_NAME=/home/stevance/oxlvra_dev/data/lvra/JSON/2026/20260202/20260202_102448.json
JSON_LAST_DIR_AND_NAME=${TEST_JSON_NAME: -34}

# Putting some data in the JSON directory
mkdir -p "$work_dir/JSON/${JSON_LAST_DIR_AND_NAME:: 14}"
cp -p $TEST_JSON_NAME $work_dir/JSON/$JSON_LAST_DIR_AND_NAME

## CODE
cd "/home/stevance/oxlvra_dev/test_$today"
cd code
mkdir -p lvra/bash
cd lvra/bash
cp -p /home/stevance/oxlvra_dev/code/lvra/bash/r0b_feature_maker.sh .
cp -p /home/stevance/oxlvra_dev/code/lvra/bash/r0b_predict.sh .
cp -p /home/stevance/oxlvra_dev/code/lvra/bash/r0b_annotator.sh .

Add data to sqlite database

This would have been done by the kafka consumer but we’re not runnign it here so we have to simulate those entries

insert into feature_making (stem, r0b) values ('20260202_102448', 0) on conflict (stem) do update set r0b=excluded.r0b;
insert into annotating (stem, r0b) values ('20260202_102448', 0) on conflict (stem) do update set r0b=excluded.r0b;