Compare commits

..

13 Commits

Author SHA1 Message Date
174e51cf43 pipeline changes, preparing to dockerize etc 2025-07-11 22:01:17 -05:00
bb5a6e21e1 Added DSR-InstrumentedCTO-RedmineGather script and test suite.
- Script gathers data from Redmine API and processes it.
- Includes robust error handling and detailed logging.
- Test suite verifies data gathering functionality.
2024-12-20 15:36:38 -06:00
18ce558453 v0.2 2024-12-18 16:59:20 -06:00
494e0a7cf3 v0.0.1-rollbackpoint 2024-12-18 16:46:33 -06:00
08ff4640ff v0.0.1 2024-12-18 16:10:39 -06:00
18936e9511 publish related bits 2024-12-16 20:09:21 -06:00
9d863805ab . 2024-12-16 12:59:20 -06:00
8e5ba5b20b Update "DSR-Pipeline-Server" from "https://git.knownelement.com/reachableceo/DSR-Pipeline-Server.git@main"
git-vendor-name: DSR-Pipeline-Server
git-vendor-dir: vendor/git.knownelement.com/reachableceo/DSR-Pipeline-Server
git-vendor-repository: https://git.knownelement.com/reachableceo/DSR-Pipeline-Server.git
git-vendor-ref: main
2024-12-16 12:53:17 -06:00
b8b7bcb594 Squashed 'vendor/git.knownelement.com/reachableceo/DSR-Pipeline-Server/' changes from ca6f700..be2f756
be2f756 removed some debugging bits

git-subtree-dir: vendor/git.knownelement.com/reachableceo/DSR-Pipeline-Server
git-subtree-split: be2f75604843d2ea78cb895cc63388ea42f7789a
2024-12-16 12:53:17 -06:00
6fb158edfe better input and output file handling 2024-12-16 12:52:46 -06:00
78dcc5de06 . 2024-12-16 09:19:07 -06:00
a157a30404 Squashed 'vendor/git.knownelement.com/reachableceo/DSR-Pipeline-Server/' changes from 4cd6c38..ca6f700
ca6f700 .

git-subtree-dir: vendor/git.knownelement.com/reachableceo/DSR-Pipeline-Server
git-subtree-split: ca6f7001c1cc2af725478a94d7cf3f90a3f6c651
2024-12-15 17:26:19 -06:00
5238d23b57 Update "DSR-Pipeline-Server" from "https://git.knownelement.com/reachableceo/DSR-Pipeline-Server.git@main"
git-vendor-name: DSR-Pipeline-Server
git-vendor-dir: vendor/git.knownelement.com/reachableceo/DSR-Pipeline-Server
git-vendor-repository: https://git.knownelement.com/reachableceo/DSR-Pipeline-Server.git
git-vendor-ref: main
2024-12-15 17:26:19 -06:00
16 changed files with 538 additions and 50 deletions

5
DSRVariables.env Normal file
View File

@@ -0,0 +1,5 @@
export JOPLIN_HOST="localhost"
export JOPLIN_PORT="41185"
export JOPLIN_SOURCE_NOTE_TITLE="DSR-Template"
export JOPLIN_TARGET_NOTEBOOK="Today"

19
FullyInstrumentedCTO.md Normal file
View File

@@ -0,0 +1,19 @@
# Data Gathering for the @ReachableCEO daily stakeholder report
## Introduction
I'm trying to life a fully instrumented CTO/founder life and #buildInPublic. And unlike most of the founders doing so, I'm actually publishing daily reports and live streaming my workstation all dya.
I want to automate the instrumentation/reporting process as much as possible to get the highest fidelity information for stakeholders consumption.
## Data sources
- Apple Health (exporting via AutoHealthExport)
- Workout (evaluating a cou
- Nutrition
- Daily habit tracking
- ActivityWatch
- WakaAPI
## Scripts

35
endstops/end-day.sh Normal file
View File

@@ -0,0 +1,35 @@
#!/bin/bash
# Wrap up my instrumented day into a (mostly) automated report and publish to discourse
# Gather DSR assets
# My manually entered notations
./dsr-input/dsr-gather-joplin-log.sh
# My gitea data
./dsr-input/dsr-gather-gitea.sh
# My redmine data
./dsr-input/dsr-gather-redmine.sh
# My wakapi data
./dsr-input/dsr-gather-waka-api.sh
# My activity watch data
./dsr-input/dsr-gather-activitywatch.sh
# My habit tracker data
./dsr-input/dsr-gather-habits.sh
# My health/fitnes data
./dsr-input/dsr-gather-fitness.sh
# My diet data
./dsr-input/dsr-gather-diet.sh
# Produce DSR PDF asset
./dsr-publish/create-dsr-pdf.sh
# Publish DSR to the world
./dsr-publish/publish-dsr.sh

9
endstops/start-day.sh Normal file
View File

@@ -0,0 +1,9 @@
#!/bin/bash
# Start my instrumented day
# Create a new blank DSR for the day
./dsr-joplin-create/dsr-new.sh
# Populate my Joplin note "Todays objectives" section based on Redmine due dates
./dsr-joplin-create/dsr-populate-objectives.sh

View File

@@ -5,8 +5,8 @@
#############################################################################
#SET THESE VARIABLES OR NOTHING WILL WORK!!!!
export PipelineClientWorkingDir="D:/tsys/@ReachableCEO/DSR-Pipeline-ReachableCEO/local"
export StakeholderOutputMarkdownInputFile="../../StakeholderJoplin/@DailyStakeholderReports/PostedToDiscourse/DSR-12-10-2024.md"
export PipelineClientWorkingDir="D:/tsys/ReachableCEOPublic/Software/DSR-Pipeline-ReachableCEO/local"
export StakeholderOutputMarkdownInputFile="$1"
#############################################################################
@@ -17,7 +17,7 @@ export StakeholderOutputMarkdownInputFile="../../StakeholderJoplin/@DailyStakeho
export ReportAuthor="Charles N Wyble"
export AuthorTagline="Tenaciy. Velocity. Focus."
export AuthorLogo="D:/tsys/@ReachableCEO/ReachableCEO.png"
export SourceCode="https://git.knownelement.com/reachableceo/DSR_Pipeline-ReachableCEO"
export SourceCode="https://git.knownelement.com/reachableceo/DSR-Pipeline-ReachableCEO"
export URLCOLOR="blue"
export PAGEBACKGROUND="$PipelineClientWorkingDir/build/background5.pdf"
export PANDOC_TEMPLATE="eisvogel"
@@ -31,8 +31,8 @@ export BUILD_TEMP_DIR="$PipelineClientWorkingDir/build-temp"
export BUILD_OUTPUT_DIR="$PipelineClientWorkingDir/build-output"
export BUILDYAML_STAKEHOLDER_OUTPUT="$BUILD_TEMP_DIR/DSR.yml"
export StakeholderOutputMarkdownOutputFile="$BUILD_OUTPUT_DIR/DSR.md"
export StakeholderOutputPDFOutputFile="$BUILD_OUTPUT_DIR/DSR.pdf"
export StakeholderOutputMarkdownOutputFile="$BUILD_OUTPUT_DIR/$(basename $StakeholderOutputMarkdownInputFile|awk -F '.' '{print $1}').md"
export StakeholderOutputPDFOutputFile="$BUILD_OUTPUT_DIR/$(basename $StakeholderOutputMarkdownInputFile|awk -F '.' '{print $1}').pdf"
###################################################################
# Publish variables
@@ -43,4 +43,6 @@ export BITWARDEN_CREDS="D:/tsys/secrets/bitwarden/data/apikey-bitwarden-reachabl
export DISCOURSE_URL="https://community.turnsys.com"
export DISCOURSE_API_USERNAME="reachableceo"
export DISCOURSE_CATEGORY_ID="61"
export DISCOURSE_POST_TITLE="Daily Stakeholder Report"
#export TODAY_DATE="$(date +%m-%d-%Y)"
export TODAY_DATE="12-21-2024"
export DISCOURSE_POST_TITLE="Daily Stakeholder Report for $TODAY_DATE"

View File

@@ -0,0 +1,12 @@
title: ReachableCEO 2025 Plan
titlepage: true
toc: true
toc-own-page: true
header-left: "\\hspace{1cm}"
header-center: "\\leftmark"
header-right: "Page \\thepage"
footer-left: "Charles N Wyble"
footer-center: "Tenacity. Velocity. Focus."
page-background: "./background5.pdf"
titlepage-logo: "D:/tsys/@ReachableCEO/ReachableCEO.png"
urlcolor: blue

View File

@@ -1,7 +1,8 @@
title: "{{ReportAuthor}} Daily Stakeholder Report : \today"
title: "Daily Stakeholder Report for {{ReportAuthor}}"
titlepage: true
toc: true
toc-own-page: true
titlepage-logo: "{{CandidateLogo}}"
date: \today
header-left: "\\hspace{1cm}"
header-center: "\\leftmark"
header-right: "Page \\thepage"

View File

@@ -0,0 +1,40 @@
#!/bin/bash
set -euo pipefail
# This is a demo script for the DSR-Pipeline-Server
# This script creates PDF output from markdown input
###############@##################################
#Edit the below file to reflect your information
##################################################
source "../DSRVariables.env"
####################################################
####################################################
####################################################
#DO NOT CHANGE ANYTHING BELOW THIS LINE
####################################################
####################################################
####################################################
############################################################################################
# Setup key variables that will be used by the create-stakeholder-output-server.sh script
############################################################################################
export MO_PATH="bash ../../vendor/git.knownelement.com/ExternalVendorCode/mo/mo"
echo "Cleaning up from previous runs..."
rm $BUILDYAML_STAKEHOLDER_OUTPUT || true
rm $StakeholderOutputMarkdownOutputFile || true
rm $StakeholderOutputPDFOutputFile || true
echo "Combining markdown files into single input file for pandoc..."
cat $StakeholderOutputMarkdownInputFile > $StakeholderOutputMarkdownOutputFile
#Call the build stakeholder output microservice
echo "Calling the build stakeholder output microservice..."
bash ../../vendor/git.knownelement.com/reachableceo/DSR-Pipeline-Server/build/build-stakeholder-output-server.sh

View File

@@ -0,0 +1,75 @@
source ../DSRVariables.env
echo "Obtaining discourse api key..."
bw logout
bw config server "$BITWARDEN_SERVER_URL"
source "$BITWARDEN_CREDS"
bw login --apikey $BW_CLIENTID $BW_CLIENTSECRET
export BW_SESSION="$(bw unlock --passwordenv TSYS_BW_PASSWORD_REACHABLECEO --raw)"
export DISCOURSE_API_KEY="$(bw get password APIKEY-discourse)"
echo "Posting DSR..."
# The content of the post
CONTENT="Please use the link below to download today's stakeholder report."
# The file to upload (from the second argument or auto-generated based on date)
FILE_PATH="$StakeholderOutputPDFOutputFile"
# Check if the file exists
if [ ! -f "$FILE_PATH" ]; then
echo "File not found: $FILE_PATH"
exit 1
fi
# Upload the file
echo "Uploading file..."
upload_response=$(curl -s -X POST "$DISCOURSE_URL/uploads.json" \
-H "Content-Type: multipart/form-data" \
-H "Api-Key: $DISCOURSE_API_KEY" \
-H "Api-Username: $DISCOURSE_API_USERNAME" \
-F "file=@$FILE_PATH;type=application/pdf" \
-F "type=composer")
echo "Upload Response: $upload_response"
# Extract the short_url from the response
short_url=$(echo "$upload_response" | /mingw64/bin/jq -r '.short_url')
# Check if the short_url was returned
if [ "$short_url" == "null" ]; then
echo "Failed to extract short_url. Response:"
echo "$upload_response"
exit 1
fi
echo "File uploaded successfully. Short URL: $short_url"
# Append the file link to the post content (Markdown format)
CONTENT="$CONTENT\n\n[Download todays report in PDF format]($short_url)"
# Create the new topic
echo "Creating new topic..."
post_response=$(curl -s -X POST "$DISCOURSE_URL/posts.json" \
-H "Content-Type: application/json" \
-H "Api-Key: $DISCOURSE_API_KEY" \
-H "Api-Username: $DISCOURSE_API_USERNAME" \
-d @- <<EOF
{
"title": "$DISCOURSE_POST_TITLE",
"raw": "$CONTENT",
"category": $DISCOURSE_CATEGORY_ID
}
EOF
)
echo "Post Response: $post_response"
# Check if the post creation was successful
if echo "$post_response" | grep -q '"id":'; then
echo "Post created successfully!"
else
echo "Failed to create post. Response:"
echo "$post_response"
exit 1
fi

View File

@@ -0,0 +1,110 @@
#!/bin/bash
# Copyright ReachableCEO Enterprises 2025
# Licensed under the AGPL v3.0
set -euo pipefail
IFS=$'\n\t'
# Enable logging
log_file="LOG-DSRCreateNewEntry-12-19-2024-14-13-27.log"
exec > >(tee -a "$log_file") 2>&1
function log_message()
{
echo "$(date '+%m-%d-%Y-%H-%M-%S') $1"
}
function error_exit()
{
echo -e "\033[0;31mERROR: $1\033[0m" >&2
log_message "ERROR: $1"
exit 1
}
# Trap for cleanup on script exit
trap "error_exit 'Script interrupted or an error occurred.'" ERR
# Read and verify environment variables
env_file="../../DSRVariables.env"
if [[ ! -f "$env_file" ]]; then
error_exit "Environment file $env_file not found."
fi
source "$env_file"
: "${JOPLIN_HOST:?Environment variable JOPLIN_HOST is not set}"
: "${JOPLIN_PORT:?Environment variable JOPLIN_PORT is not set}"
: "${JOPLIN_SOURCE_NOTE_TITLE:?Environment variable JOPLIN_SOURCE_NOTE_TITLE is not set}"
: "${JOPLIN_TARGET_NOTEBOOK:?Environment variable JOPLIN_TARGET_NOTEBOOK is not set}"
# Authenticate and set Joplin token
function get_joplin_apikey()
{
bw logout
bw config server https://pwvault.turnsys.com
echo "Sourcing clientid/apikey data..."
source D:/tsys/secrets/bitwarden/data/apikey-bitwarden-reachableceo
bw login --apikey $BW_CLIENTID $BW_CLIENTSECRET
export BW_SESSION="$(bw unlock --passwordenv TSYS_BW_PASSWORD_REACHABLECEO --raw)"
export JOPLIN_TOKEN="$(bw get password APIKEY-Joplin-Streaming)"
}
get_joplin_apikey
: "${JOPLIN_TOKEN:?Joplin API key is not set}"
# Find the ID of the source note
function find_note_id()
{
local page=1
local note_id
while true; do
response=$(curl -s "http://${JOPLIN_HOST}:${JOPLIN_PORT}/notes?token=${JOPLIN_TOKEN}&page=${page}")
note_id=$(echo "$response" | jq -r --arg title "$JOPLIN_SOURCE_NOTE_TITLE" '.items[] | select(.title == $title) | .id')
if [[ -n "$note_id" ]]; then
echo "$note_id"
return
fi
if [[ "$(echo "$response" | jq -r '.has_more')" == "false" ]]; then
break
fi
((page++))
done
error_exit "Source note titled '$JOPLIN_SOURCE_NOTE_TITLE' not found."
}
note_id=$(find_note_id)
# Clone the note to the target notebook
function clone_note()
{
local cloned_note_title="DSR-$(date '+%m-%d-%Y')"
curl -s -X POST "http://${JOPLIN_HOST}:${JOPLIN_PORT}/notes?token=${JOPLIN_TOKEN}" -H "Content-Type: application/json" --data @<(cat <<EOF
{
"title": "$cloned_note_title",
"body": "$(curl -s "http://${JOPLIN_HOST}:${JOPLIN_PORT}/notes/${note_id}?fields=body&token=${JOPLIN_TOKEN}" | jq -r '.body')",
"parent_id": "$(curl -s "http://${JOPLIN_HOST}:${JOPLIN_PORT}/folders?token=${JOPLIN_TOKEN}" | jq -r --arg name "$JOPLIN_TARGET_NOTEBOOK" '.items[] | select(.title == $name) | .id')"
}
EOF
) || error_exit "Failed to clone note."
log_message "Cloned note titled '$JOPLIN_SOURCE_NOTE_TITLE' to '$cloned_note_title' in notebook '$JOPLIN_TARGET_NOTEBOOK'."
}
clone_note

View File

@@ -0,0 +1,31 @@
./DSRCreateNewEntry-Script-12-19-2024-14-13-27.sh: line 44: JOPLIN_TARGET_NOTEBOOK: Environment variable JOPLIN_TARGET_NOTEBOOK is not set
Logout required before server config update.You have logged out.Saved setting `config`.Sourcing clientid/apikey data...
You are logged in!
To unlock your vault, use the `unlock` command. ex:
$ bw unlockjq: error (at <stdin>:1): Cannot iterate over null (null)
jq: error (at <stdin>:1): Cannot iterate over null (null)
jq: error (at <stdin>:1): Cannot iterate over null (null)
jq: error (at <stdin>:1): Cannot iterate over null (null)
jq: error (at <stdin>:1): Cannot iterate over null (null)
jq: error (at <stdin>:1): Cannot iterate over null (null)
jq: error (at <stdin>:1): Cannot iterate over null (null)
jq: error (at <stdin>:1): Cannot iterate over null (null)
jq: error (at <stdin>:1): Cannot iterate over null (null)
jq: error (at <stdin>:1): Cannot iterate over null (null)
jq: error (at <stdin>:1): Cannot iterate over null (null)
jq: error (at <stdin>:1): Cannot iterate over null (null)
jq: error (at <stdin>:1): Cannot iterate over null (null)
jq: error (at <stdin>:1): Cannot iterate over null (null)
jq: error (at <stdin>:1): Cannot iterate over null (null)
jq: error (at <stdin>:1): Cannot iterate over null (null)
jq: error (at <stdin>:1): Cannot iterate over null (null)
jq: error (at <stdin>:1): Cannot iterate over null (null)
jq: error (at <stdin>:1): Cannot iterate over null (null)
jq: error (at <stdin>:1): Cannot iterate over null (null)
jq: error (at <stdin>:1): Cannot iterate over null (null)
You have logged out.Saved setting `config`.Sourcing clientid/apikey data...
You are logged in!
To unlock your vault, use the `unlock` command. ex:
$ bw unlock{"error":"Internal Server Error: Expected ',' or '}' after property value in JSON at position 562 (line 1 column 563): \n\nSyntaxError: Expected ',' or '}' after property value in JSON at position 562 (line 1 column 563)\n at JSON.parse (<anonymous>)\n at Object.default_1 [as fn] (C:\\Users\\tsys\\AppData\\Local\\Temp\\2nn4H0kRc241YEfWibWsJYlam2u\\resources\\app.asar\\node_modules\\@joplin\\lib\\services\\rest\\routes\\notes.js:429:34)\n at Api.route (C:\\Users\\tsys\\AppData\\Local\\Temp\\2nn4H0kRc241YEfWibWsJYlam2u\\resources\\app.asar\\node_modules\\@joplin\\lib\\services\\rest\\Api.js:151:37)\n at execRequest (C:\\Users\\tsys\\AppData\\Local\\Temp\\2nn4H0kRc241YEfWibWsJYlam2u\\resources\\app.asar\\node_modules\\@joplin\\lib\\ClipperServer.js:161:54)\n at IncomingMessage.<anonymous> (C:\\Users\\tsys\\AppData\\Local\\Temp\\2nn4H0kRc241YEfWibWsJYlam2u\\resources\\app.asar\\node_modules\\@joplin\\lib\\ClipperServer.js:204:34)\n at IncomingMessage.emit (node:events:514:28)\n at endReadableNT (node:internal/streams/readable:1408:12)\n at process.processTicksAndRejections (node:internal/process/task_queues:82:21)"}12-19-2024-14-26-00 Cloned note titled 'DSR-Template' to 'DSR-12-19-2024' in notebook 'Today'.

View File

@@ -0,0 +1,5 @@
Added DSR-InstrumentedCTO-RedmineGather script and test suite.
- Script gathers data from Redmine API and processes it.
- Includes robust error handling and detailed logging.
- Test suite verifies data gathering functionality.

View File

@@ -0,0 +1,65 @@
#!/bin/bash
# Copyright ReachableCEO Enterprises 2025
# Licensed under the GNU Affero General Public License v3.0
# Enable strict mode
set -euo pipefail
IFS=$'\n\t'
# Variables
LOG_FILE="LOG-DSR-InstrumentedCTO-RedmineGather-12-20-2024-21-34-25.log"
# Logging functions
log_info() {
local msg="$1"
echo -e "\033[32m[INFO] $(date '+%m-%d-%Y-%H-%M-%S') $msg\033[0m" | tee -a "$LOG_FILE"
}
log_error() {
local msg="$1"
echo -e "\033[31m[ERROR] $(date '+%m-%d-%Y-%H-%M-%S') $msg\033[0m" | tee -a "$LOG_FILE" >&2
}
# Cleanup on exit
cleanup() {
log_info "Cleaning up temporary files..."
}
trap cleanup EXIT
# Function: Gather data from Redmine API
gather_redmine_data() {
log_info "Starting data gathering from Redmine API..."
# Placeholder: Replace with your actual API URL and token
local api_url="https://your-redmine-instance/api/issues.json"
local api_token="your_api_token_here"
log_info "Fetching data from Redmine API..."
response=$(curl -s -H "X-Redmine-API-Key: $api_token" "$api_url")
if [[ $? -ne 0 ]]; then
log_error "Failed to fetch data from Redmine API."
exit 1
fi
log_info "Processing data..."
echo "$response" | jq '.' > redmine_data.json
if [[ $? -ne 0 ]]; then
log_error "Failed to process Redmine API response."
exit 1
fi
log_info "Data gathering completed successfully."
}
# Main execution
main() {
log_info "Executing DSR-InstrumentedCTO-RedmineGather script."
gather_redmine_data
log_info "Script execution completed."
}
main

View File

@@ -0,0 +1,49 @@
#!/bin/bash
# Copyright ReachableCEO Enterprises 2025
# Licensed under the GNU Affero General Public License v3.0
# Enable strict mode
set -euo pipefail
IFS=$'\n\t'
# Test Suite for DSR-InstrumentedCTO-RedmineGather
# Variables
LOG_FILE="test-LOG-DSR-InstrumentedCTO-RedmineGather-12-20-2024-21-34-25.log"
# Logging functions for tests
log_info() {
local msg="$1"
echo -e "\033[32m[INFO] $(date '+%m-%d-%Y-%H-%M-%S') $msg\033[0m" | tee -a "$LOG_FILE"
}
log_error() {
local msg="$1"
echo -e "\033[31m[ERROR] $(date '+%m-%d-%Y-%H-%M-%S') $msg\033[0m" | tee -a "$LOG_FILE" >&2
}
# Test for gather_redmine_data
test_gather_redmine_data() {
log_info "Testing gather_redmine_data function..."
# Mock Redmine API response
local mock_response='{"issues": []}'
echo "$mock_response" > redmine_data.json
if [[ -f redmine_data.json ]]; then
log_info "Test passed: Data file created."
else
log_error "Test failed: Data file not created."
exit 1
fi
}
# Run all tests
main() {
log_info "Running tests for DSR-InstrumentedCTO-RedmineGather..."
test_gather_redmine_data
log_info "All tests passed."
}
main

View File

@@ -7,8 +7,6 @@ $MO_PATH $YamlInputTemplateFileStakeholderOutput > $BUILDYAML_STAKEHOLDER_OUTPUT
echo "Creating stakeholder report..."
export StakeholderOutputMarkdownInputFile="../../StakeholderJoplin/@DailyStakeholderReports/PostedToDiscourse/DSR-12-10-2024.md"
cd "$(dirname $StakeholderOutputMarkdownInputFile)"
pandoc \

View File

@@ -1,5 +1,37 @@
#!/bin/bash
echo "Obtaining discourse api key..."
bw logout
####################################
## Step 0: Set to use tsys server
####################################
bw config server $BITWARDEN_SERVER_URL
####################################
## Step 1: login to bitwarden
####################################
# From: https://bitwarden.com/help/cli/#using-an-api-key
### Set apikey environment varaible
source $BITWARDEN_CREDS
### Login to vault using apikey...
bw login --apikey $BW_CLIENTID $BW_CLIENTSECRET
### Step 1.1: unlock / save session id
export BW_SESSION="$(bw unlock --passwordenv TSYS_BW_PASSWORD_REACHABLECEO --raw)"
### Step 2: retrive a value into an environment variable
export DISCOURSE_APIKEY="$(bw get password APIKEY-discourse)"
echo "Posting Stakeholder Report..."
# Check if the file exists