pipeline changes, preparing to dockerize etc

- Script gathers data from Redmine API and processes it.
- Includes robust error handling and detailed logging.
- Test suite verifies data gathering functionality.

.gitignore

@@ -0,0 +1,7 @@
+local/build-temp/*.md
+local/build-temp/*.yml
+
+local/build-output/*.md
+local/build-output/*.pdf
+
+StakeholderJoplin/*

DSRVariables.env

@@ -0,0 +1,5 @@
+export JOPLIN_HOST="localhost"
+export JOPLIN_PORT="41185"
+
+export JOPLIN_SOURCE_NOTE_TITLE="DSR-Template"
+export JOPLIN_TARGET_NOTEBOOK="Today"

FullyInstrumentedCTO.md

@@ -0,0 +1,19 @@
+# Data Gathering for the @ReachableCEO daily stakeholder report
+
+## Introduction
+
+I'm trying to life a fully instrumented CTO/founder life and #buildInPublic. And unlike most of the founders doing so, I'm actually publishing daily reports and live streaming my workstation all dya.
+
+I want to automate the instrumentation/reporting process as much as possible to get the highest fidelity information for stakeholders consumption.
+
+## Data sources
+
+- Apple Health (exporting via AutoHealthExport)
+- Workout (evaluating a cou
+- Nutrition
+- Daily habit tracking
+- ActivityWatch
+- WakaAPI
+
+
+## Scripts

README.md

@@ -1,2 +1,47 @@
-# DSR-Pipeline-ReachableCEO
+# DailyStakeholderReport-Pipeline
 
+- [DailyStakeholderReport-Pipeline](#dailystakeholderreport-pipeline)
+  - [Introduction](#introduction)
+  - [Scripts](#scripts)
+    - [Start a new day](#start-a-new-day)
+      - [Create a new (blank) DSR](#create-a-new-blank-dsr)
+      - [Populate the new DSR with objectives for the day](#populate-the-new-dsr-with-objectives-for-the-day)
+    - [End the day](#end-the-day)
+      - [Collating and Publishing the report](#collating-and-publishing-the-report)
+      - [Gathering data for the report](#gathering-data-for-the-report)
+
+## Introduction
+
+I publish a daily stakeholder report to the TSYS Group discourse every day. Up until 11/29 I was performing the publishing step manually. On 11/29 I automated the pdf creation and publishing to discourse procedure.
+
+I also begin work on automated gathering of data going into the new version of the report that is in development.
+
+## Scripts
+
+### Start a new day
+
+Start a new day. See the script: **endstops/start-day.sh**
+
+#### Create a new (blank) DSR
+
+See the script: **dsr-joplin-create/dsr-new.sh**
+
+#### Populate the new DSR with objectives for the day
+
+See the script: **dsr-joplin-create/dsr-populate-objectives.sh**
+
+### End the day
+
+End the day. See the script: **endstops/end-day.sh**
+
+#### Collating and Publishing the report
+
+Publishing the report involves three steps:
+
+1. Exporting the markdown from Joplin (and any attachments (i often take screenshots throughout the day)) I am still doing this manually due to having issues installing the Joplin CLI.
+
+2. Creating a (formatted/styled) PDF from the exported markdown/attachments. See the script: **dsr-publish/create-dsr-pdf.sh**.
+
+3. Uploading the PDF to discourse and creating a new topic in the appropriate category. See the script: **dsr-publish/publish-dsr.sh**
+
+#### Gathering data for the report

endstops/end-day.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Wrap up my instrumented day into a (mostly) automated report and publish to discourse
+
+# Gather DSR assets
+
+# My manually entered notations
+./dsr-input/dsr-gather-joplin-log.sh
+
+# My gitea data
+./dsr-input/dsr-gather-gitea.sh
+
+# My redmine data
+./dsr-input/dsr-gather-redmine.sh
+
+# My wakapi data
+./dsr-input/dsr-gather-waka-api.sh
+
+# My activity watch data
+./dsr-input/dsr-gather-activitywatch.sh
+
+# My habit tracker data
+./dsr-input/dsr-gather-habits.sh
+
+# My health/fitnes data 
+./dsr-input/dsr-gather-fitness.sh
+
+# My diet data
+./dsr-input/dsr-gather-diet.sh
+
+# Produce DSR PDF asset
+./dsr-publish/create-dsr-pdf.sh
+
+# Publish DSR to the world
+./dsr-publish/publish-dsr.sh

endstops/start-day.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# Start my instrumented day 
+
+# Create a new blank DSR for the day 
+./dsr-joplin-create/dsr-new.sh
+
+# Populate my Joplin note "Todays objectives" section based on Redmine due dates
+./dsr-joplin-create/dsr-populate-objectives.sh

local/DSRVariables.env

@@ -0,0 +1,48 @@
+###################################################
+# Modify these values to suit
+###################################################
+
+#############################################################################
+#SET THESE VARIABLES OR NOTHING WILL WORK!!!!
+
+export PipelineClientWorkingDir="D:/tsys/ReachableCEOPublic/Software/DSR-Pipeline-ReachableCEO/local"
+export StakeholderOutputMarkdownInputFile="$1" 
+
+#############################################################################
+
+##########################################
+# Layout/title page /formatting options
+##########################################
+
+export ReportAuthor="Charles N Wyble"
+export AuthorTagline="Tenaciy. Velocity. Focus."
+export AuthorLogo="D:/tsys/@ReachableCEO/ReachableCEO.png"
+export SourceCode="https://git.knownelement.com/reachableceo/DSR-Pipeline-ReachableCEO"
+export URLCOLOR="blue"
+export PAGEBACKGROUND="$PipelineClientWorkingDir/build/background5.pdf"
+export PANDOC_TEMPLATE="eisvogel"
+export YamlInputTemplateFileStakeholderOutput="$PipelineClientWorkingDir/build/BuildTemplate-DSR.yml"
+
+################################################################### 
+# Output Artifact variables
+################################################################### 
+
+export BUILD_TEMP_DIR="$PipelineClientWorkingDir/build-temp"
+export BUILD_OUTPUT_DIR="$PipelineClientWorkingDir/build-output"
+export BUILDYAML_STAKEHOLDER_OUTPUT="$BUILD_TEMP_DIR/DSR.yml"
+
+export StakeholderOutputMarkdownOutputFile="$BUILD_OUTPUT_DIR/$(basename $StakeholderOutputMarkdownInputFile|awk -F '.' '{print $1}').md" 
+export StakeholderOutputPDFOutputFile="$BUILD_OUTPUT_DIR/$(basename $StakeholderOutputMarkdownInputFile|awk -F '.' '{print $1}').pdf" 
+
+################################################################### 
+# Publish variables
+################################################################### 
+
+export BITWARDEN_SERVER_URL="https://pwvault.turnsys.com"
+export BITWARDEN_CREDS="D:/tsys/secrets/bitwarden/data/apikey-bitwarden-reachableceo"
+export DISCOURSE_URL="https://community.turnsys.com"
+export DISCOURSE_API_USERNAME="reachableceo"
+export DISCOURSE_CATEGORY_ID="61"
+#export TODAY_DATE="$(date +%m-%d-%Y)"
+export TODAY_DATE="12-21-2024"
+export DISCOURSE_POST_TITLE="Daily Stakeholder Report for $TODAY_DATE"

local/build/BuildTemplate-2025Plan.yml

@@ -0,0 +1,12 @@
+title: ReachableCEO 2025 Plan
+titlepage: true
+toc: true
+toc-own-page: true
+header-left: "\\hspace{1cm}"
+header-center: "\\leftmark"
+header-right: "Page \\thepage"
+footer-left: "Charles N Wyble"
+footer-center: "Tenacity. Velocity. Focus."
+page-background: "./background5.pdf"
+titlepage-logo: "D:/tsys/@ReachableCEO/ReachableCEO.png"
+urlcolor: blue

local/build/BuildTemplate-DSR.yml

@@ -0,0 +1,13 @@
+title: "Daily Stakeholder Report for {{ReportAuthor}}"
+titlepage: true
+toc: true
+toc-own-page: true
+titlepage-logo: "{{CandidateLogo}}"
+header-left: "\\hspace{1cm}"
+header-center: "\\leftmark"
+header-right: "Page \\thepage"
+footer-left: "{{ReportAuthor}}"
+footer-center: "{{AuthorTagline}}"
+footer-right: "[Pipeline Source code]({{SourceCode}})"
+urlcolor: {{URLCOLOR}}
+page-background: "{{PAGEBACKGROUND}}"

local/build/background5.pdf

@@ -0,0 +1,70 @@
+%PDF-1.5
+%<25><><EFBFBD><EFBFBD>
+3 0 obj
+<< /Length 4 0 R
+   /Filter /FlateDecode
+>>
+stream
+x<EFBFBD>m<EFBFBD>;<0E>@D<>=<3D>/<2F><><EFBFBD><EFBFBD><EFBFBD>18	<14><>K삢D
+r<EFBFBD><EFBFBD><EFBFBD><EFBFBD>Ư<EFBFBD>0f<EFBFBD><EFBFBD>|%<25>߅ <20>1<EFBFBD><31><EFBFBD><EFBFBD>D<EFBFBD><44>g,7<>
+!<21>i<03>jҔ<6A>P	<09>/c<>h<06>px<70>r‰<72><C289><EFBFBD><EFBFBD>1W<31>^<5E><>Q<EFBFBD>d<>%q<>!<21><>9ZD<12>j<EFBFBD><6A>
+U<EFBFBD>Ѫ<EFBFBD><EFBFBD><EFBFBD><EFBFBD>A<EFBFBD>p<EFBFBD><EFBFBD>r$<24>&]x_<78><5F>ʑȽ<CA91>I<EFBFBD>W<><7F>?f<>r)<1F>6EU
+endstream
+endobj
+4 0 obj
+   168
+endobj
+2 0 obj
+<<
+   /ExtGState <<
+      /a0 << /CA 1 /ca 1 >>
+   >>
+>>
+endobj
+5 0 obj
+<< /Type /Page
+   /Parent 1 0 R
+   /MediaBox [ 0 0 595.275574 841.889771 ]
+   /Contents 3 0 R
+   /Group <<
+      /Type /Group
+      /S /Transparency
+      /I true
+      /CS /DeviceRGB
+   >>
+   /Resources 2 0 R
+>>
+endobj
+1 0 obj
+<< /Type /Pages
+   /Kids [ 5 0 R ]
+   /Count 1
+>>
+endobj
+6 0 obj
+<< /Creator (cairo 1.14.8 (http://cairographics.org))
+   /Producer (cairo 1.14.8 (http://cairographics.org))
+>>
+endobj
+7 0 obj
+<< /Type /Catalog
+   /Pages 1 0 R
+>>
+endobj
+xref
+0 8
+0000000000 65535 f 
+0000000582 00000 n 
+0000000282 00000 n 
+0000000015 00000 n 
+0000000260 00000 n 
+0000000354 00000 n 
+0000000647 00000 n 
+0000000774 00000 n 
+trailer
+<< /Size 8
+   /Root 7 0 R
+   /Info 6 0 R
+>>
+startxref
+826

local/build/build-2025Plan.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# This is a demo script for the DSR-Pipeline-Server
+# This script creates PDF output from markdown input
+
+
+###############@##################################
+#Edit the below file to reflect your information
+##################################################
+
+source "../DSRVariables.env"
+
+####################################################
+####################################################
+####################################################
+#DO NOT CHANGE ANYTHING BELOW THIS LINE
+####################################################
+####################################################
+####################################################
+
+############################################################################################
+# Setup key variables that will be used by the create-stakeholder-output-server.sh script
+############################################################################################
+
+export MO_PATH="bash ../../vendor/git.knownelement.com/ExternalVendorCode/mo/mo"
+
+echo "Cleaning up from previous runs..."
+
+rm $BUILDYAML_STAKEHOLDER_OUTPUT || true
+rm $StakeholderOutputMarkdownOutputFile || true
+rm $StakeholderOutputPDFOutputFile || true
+
+echo "Combining markdown files into single input file for pandoc..."
+cat $StakeholderOutputMarkdownInputFile > $StakeholderOutputMarkdownOutputFile
+
+#Call the build stakeholder output microservice
+echo "Calling the build stakeholder output microservice..."
+bash ../../vendor/git.knownelement.com/reachableceo/DSR-Pipeline-Server/build/build-stakeholder-output-server.sh

local/build/build-dsr.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# This is a demo script for the DSR-Pipeline-Server
+# This script creates PDF output from markdown input
+
+
+###############@##################################
+#Edit the below file to reflect your information
+##################################################
+
+source "../DSRVariables.env"
+
+####################################################
+####################################################
+####################################################
+#DO NOT CHANGE ANYTHING BELOW THIS LINE
+####################################################
+####################################################
+####################################################
+
+############################################################################################
+# Setup key variables that will be used by the create-stakeholder-output-server.sh script
+############################################################################################
+
+export MO_PATH="bash ../../vendor/git.knownelement.com/ExternalVendorCode/mo/mo"
+
+echo "Cleaning up from previous runs..."
+
+rm $BUILDYAML_STAKEHOLDER_OUTPUT || true
+rm $StakeholderOutputMarkdownOutputFile || true
+rm $StakeholderOutputPDFOutputFile || true
+
+echo "Combining markdown files into single input file for pandoc..."
+cat $StakeholderOutputMarkdownInputFile > $StakeholderOutputMarkdownOutputFile
+
+#Call the build stakeholder output microservice
+echo "Calling the build stakeholder output microservice..."
+bash ../../vendor/git.knownelement.com/reachableceo/DSR-Pipeline-Server/build/build-stakeholder-output-server.sh

local/build/publish-dsr.sh

@@ -0,0 +1,75 @@
+source ../DSRVariables.env
+
+echo "Obtaining discourse api key..."
+bw logout
+bw config server "$BITWARDEN_SERVER_URL"
+source "$BITWARDEN_CREDS"
+bw login --apikey $BW_CLIENTID $BW_CLIENTSECRET
+export BW_SESSION="$(bw unlock --passwordenv TSYS_BW_PASSWORD_REACHABLECEO --raw)"
+export DISCOURSE_API_KEY="$(bw get password APIKEY-discourse)"
+
+echo "Posting DSR..."
+
+# The content of the post
+CONTENT="Please use the link below to download today's stakeholder report."
+
+# The file to upload (from the second argument or auto-generated based on date)
+FILE_PATH="$StakeholderOutputPDFOutputFile"
+
+# Check if the file exists
+if [ ! -f "$FILE_PATH" ]; then
+  echo "File not found: $FILE_PATH"
+  exit 1
+fi
+
+# Upload the file
+echo "Uploading file..."
+upload_response=$(curl -s -X POST "$DISCOURSE_URL/uploads.json" \
+  -H "Content-Type: multipart/form-data" \
+  -H "Api-Key: $DISCOURSE_API_KEY" \
+  -H "Api-Username: $DISCOURSE_API_USERNAME" \
+  -F "file=@$FILE_PATH;type=application/pdf" \
+  -F "type=composer")
+
+echo "Upload Response: $upload_response"
+
+# Extract the short_url from the response
+short_url=$(echo "$upload_response" | /mingw64/bin/jq -r '.short_url')
+
+# Check if the short_url was returned
+if [ "$short_url" == "null" ]; then
+  echo "Failed to extract short_url. Response:"
+  echo "$upload_response"
+  exit 1
+fi
+
+echo "File uploaded successfully. Short URL: $short_url"
+
+# Append the file link to the post content (Markdown format)
+CONTENT="$CONTENT\n\n[Download todays report in PDF format]($short_url)"
+
+# Create the new topic
+echo "Creating new topic..."
+post_response=$(curl -s -X POST "$DISCOURSE_URL/posts.json" \
+  -H "Content-Type: application/json" \
+  -H "Api-Key: $DISCOURSE_API_KEY" \
+  -H "Api-Username: $DISCOURSE_API_USERNAME" \
+  -d @- <<EOF
+{
+  "title": "$DISCOURSE_POST_TITLE",
+  "raw": "$CONTENT",
+  "category": $DISCOURSE_CATEGORY_ID
+}
+EOF
+)
+
+echo "Post Response: $post_response"
+
+# Check if the post creation was successful
+if echo "$post_response" | grep -q '"id":'; then
+  echo "Post created successfully!"
+else
+  echo "Failed to create post. Response:"
+  echo "$post_response"
+  exit 1
+fi

microservices/DSRCreateNewEntry/DSRCreateNewEntry-Script-12-19-2024-14-13-27.sh

@@ -0,0 +1,110 @@
+#!/bin/bash
+# Copyright ReachableCEO Enterprises 2025
+# Licensed under the AGPL v3.0
+
+set -euo pipefail
+IFS=$'\n\t'
+
+# Enable logging
+log_file="LOG-DSRCreateNewEntry-12-19-2024-14-13-27.log"
+exec > >(tee -a "$log_file") 2>&1
+
+function log_message()
+
+{
+
+    echo "$(date '+%m-%d-%Y-%H-%M-%S') $1"
+
+}
+
+function error_exit()
+
+{
+
+    echo -e "\033[0;31mERROR: $1\033[0m" >&2
+    log_message "ERROR: $1"
+    exit 1
+
+}
+
+# Trap for cleanup on script exit
+trap "error_exit 'Script interrupted or an error occurred.'" ERR
+
+# Read and verify environment variables
+env_file="../../DSRVariables.env"
+if [[ ! -f "$env_file" ]]; then
+    error_exit "Environment file $env_file not found."
+fi
+
+source "$env_file"
+
+: "${JOPLIN_HOST:?Environment variable JOPLIN_HOST is not set}"
+: "${JOPLIN_PORT:?Environment variable JOPLIN_PORT is not set}"
+: "${JOPLIN_SOURCE_NOTE_TITLE:?Environment variable JOPLIN_SOURCE_NOTE_TITLE is not set}"
+: "${JOPLIN_TARGET_NOTEBOOK:?Environment variable JOPLIN_TARGET_NOTEBOOK is not set}"
+
+# Authenticate and set Joplin token
+function get_joplin_apikey()
+
+{
+
+    bw logout
+    bw config server https://pwvault.turnsys.com
+    echo "Sourcing clientid/apikey data..."
+    source D:/tsys/secrets/bitwarden/data/apikey-bitwarden-reachableceo
+    bw login --apikey $BW_CLIENTID $BW_CLIENTSECRET
+    export BW_SESSION="$(bw unlock --passwordenv TSYS_BW_PASSWORD_REACHABLECEO --raw)"
+    export JOPLIN_TOKEN="$(bw get password APIKEY-Joplin-Streaming)"
+
+}
+
+get_joplin_apikey
+
+: "${JOPLIN_TOKEN:?Joplin API key is not set}"
+
+# Find the ID of the source note
+function find_note_id()
+
+{
+
+    local page=1
+    local note_id
+    while true; do
+        response=$(curl -s "http://${JOPLIN_HOST}:${JOPLIN_PORT}/notes?token=${JOPLIN_TOKEN}&page=${page}")
+        note_id=$(echo "$response" | jq -r --arg title "$JOPLIN_SOURCE_NOTE_TITLE" '.items[] | select(.title == $title) | .id')
+        if [[ -n "$note_id" ]]; then
+            echo "$note_id"
+            return
+        fi
+        if [[ "$(echo "$response" | jq -r '.has_more')" == "false" ]]; then
+            break
+        fi
+        ((page++))
+    done
+
+    error_exit "Source note titled '$JOPLIN_SOURCE_NOTE_TITLE' not found."
+
+}
+
+note_id=$(find_note_id)
+
+# Clone the note to the target notebook
+function clone_note()
+
+{
+
+    local cloned_note_title="DSR-$(date '+%m-%d-%Y')"
+    curl -s -X POST "http://${JOPLIN_HOST}:${JOPLIN_PORT}/notes?token=${JOPLIN_TOKEN}"         -H "Content-Type: application/json"         --data @<(cat <<EOF
+{
+    "title": "$cloned_note_title",
+    "body": "$(curl -s "http://${JOPLIN_HOST}:${JOPLIN_PORT}/notes/${note_id}?fields=body&token=${JOPLIN_TOKEN}" | jq -r '.body')",
+    "parent_id": "$(curl -s "http://${JOPLIN_HOST}:${JOPLIN_PORT}/folders?token=${JOPLIN_TOKEN}" | jq -r --arg name "$JOPLIN_TARGET_NOTEBOOK" '.items[] | select(.title == $name) | .id')"
+}
+EOF
+        ) || error_exit "Failed to clone note."
+
+    log_message "Cloned note titled '$JOPLIN_SOURCE_NOTE_TITLE' to '$cloned_note_title' in notebook '$JOPLIN_TARGET_NOTEBOOK'."
+
+}
+
+clone_note

microservices/DSRCreateNewEntry/LOG-DSRCreateNewEntry-12-19-2024-14-13-27.log

@@ -0,0 +1,31 @@
+./DSRCreateNewEntry-Script-12-19-2024-14-13-27.sh: line 44: JOPLIN_TARGET_NOTEBOOK: Environment variable JOPLIN_TARGET_NOTEBOOK is not set
+Logout required before server config update.You have logged out.Saved setting `config`.Sourcing clientid/apikey data...
+You are logged in!
+
+To unlock your vault, use the `unlock` command. ex:
+$ bw unlockjq: error (at <stdin>:1): Cannot iterate over null (null)
+jq: error (at <stdin>:1): Cannot iterate over null (null)
+jq: error (at <stdin>:1): Cannot iterate over null (null)
+jq: error (at <stdin>:1): Cannot iterate over null (null)
+jq: error (at <stdin>:1): Cannot iterate over null (null)
+jq: error (at <stdin>:1): Cannot iterate over null (null)
+jq: error (at <stdin>:1): Cannot iterate over null (null)
+jq: error (at <stdin>:1): Cannot iterate over null (null)
+jq: error (at <stdin>:1): Cannot iterate over null (null)
+jq: error (at <stdin>:1): Cannot iterate over null (null)
+jq: error (at <stdin>:1): Cannot iterate over null (null)
+jq: error (at <stdin>:1): Cannot iterate over null (null)
+jq: error (at <stdin>:1): Cannot iterate over null (null)
+jq: error (at <stdin>:1): Cannot iterate over null (null)
+jq: error (at <stdin>:1): Cannot iterate over null (null)
+jq: error (at <stdin>:1): Cannot iterate over null (null)
+jq: error (at <stdin>:1): Cannot iterate over null (null)
+jq: error (at <stdin>:1): Cannot iterate over null (null)
+jq: error (at <stdin>:1): Cannot iterate over null (null)
+jq: error (at <stdin>:1): Cannot iterate over null (null)
+jq: error (at <stdin>:1): Cannot iterate over null (null)
+You have logged out.Saved setting `config`.Sourcing clientid/apikey data...
+You are logged in!
+
+To unlock your vault, use the `unlock` command. ex:
+$ bw unlock{"error":"Internal Server Error: Expected ',' or '}' after property value in JSON at position 562 (line 1 column 563): \n\nSyntaxError: Expected ',' or '}' after property value in JSON at position 562 (line 1 column 563)\n    at JSON.parse (<anonymous>)\n    at Object.default_1 [as fn] (C:\\Users\\tsys\\AppData\\Local\\Temp\\2nn4H0kRc241YEfWibWsJYlam2u\\resources\\app.asar\\node_modules\\@joplin\\lib\\services\\rest\\routes\\notes.js:429:34)\n    at Api.route (C:\\Users\\tsys\\AppData\\Local\\Temp\\2nn4H0kRc241YEfWibWsJYlam2u\\resources\\app.asar\\node_modules\\@joplin\\lib\\services\\rest\\Api.js:151:37)\n    at execRequest (C:\\Users\\tsys\\AppData\\Local\\Temp\\2nn4H0kRc241YEfWibWsJYlam2u\\resources\\app.asar\\node_modules\\@joplin\\lib\\ClipperServer.js:161:54)\n    at IncomingMessage.<anonymous> (C:\\Users\\tsys\\AppData\\Local\\Temp\\2nn4H0kRc241YEfWibWsJYlam2u\\resources\\app.asar\\node_modules\\@joplin\\lib\\ClipperServer.js:204:34)\n    at IncomingMessage.emit (node:events:514:28)\n    at endReadableNT (node:internal/streams/readable:1408:12)\n    at process.processTicksAndRejections (node:internal/process/task_queues:82:21)"}12-19-2024-14-26-00 Cloned note titled 'DSR-Template' to 'DSR-12-19-2024' in notebook 'Today'.

microservices/DSRInstrumentedCTORedmineGather/DSR-InstrumentedCTO-RedmineGather-12-20-2024-21-34-25-GitMsg.txt

@@ -0,0 +1,5 @@
+Added DSR-InstrumentedCTO-RedmineGather script and test suite.
+
+- Script gathers data from Redmine API and processes it.
+- Includes robust error handling and detailed logging.
+- Test suite verifies data gathering functionality.

microservices/DSRInstrumentedCTORedmineGather/DSR-InstrumentedCTO-RedmineGather-12-20-2024-21-34-25-Script.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+# Copyright ReachableCEO Enterprises 2025
+# Licensed under the GNU Affero General Public License v3.0
+
+# Enable strict mode
+set -euo pipefail
+IFS=$'\n\t'
+
+# Variables
+LOG_FILE="LOG-DSR-InstrumentedCTO-RedmineGather-12-20-2024-21-34-25.log"
+
+# Logging functions
+log_info() {
+    local msg="$1"
+    echo -e "\033[32m[INFO] $(date '+%m-%d-%Y-%H-%M-%S') $msg\033[0m" | tee -a "$LOG_FILE"
+}
+
+log_error() {
+    local msg="$1"
+    echo -e "\033[31m[ERROR] $(date '+%m-%d-%Y-%H-%M-%S') $msg\033[0m" | tee -a "$LOG_FILE" >&2
+}
+
+# Cleanup on exit
+cleanup() {
+    log_info "Cleaning up temporary files..."
+}
+trap cleanup EXIT
+
+# Function: Gather data from Redmine API
+gather_redmine_data() {
+
+    log_info "Starting data gathering from Redmine API..."
+
+    # Placeholder: Replace with your actual API URL and token
+    local api_url="https://your-redmine-instance/api/issues.json"
+    local api_token="your_api_token_here"
+
+    log_info "Fetching data from Redmine API..."
+    response=$(curl -s -H "X-Redmine-API-Key: $api_token" "$api_url")
+    if [[ $? -ne 0 ]]; then
+        log_error "Failed to fetch data from Redmine API."
+        exit 1
+    fi
+
+    log_info "Processing data..."
+    echo "$response" | jq '.' > redmine_data.json
+    if [[ $? -ne 0 ]]; then
+        log_error "Failed to process Redmine API response."
+        exit 1
+    fi
+
+    log_info "Data gathering completed successfully."
+}
+
+# Main execution
+main() {
+
+    log_info "Executing DSR-InstrumentedCTO-RedmineGather script."
+    gather_redmine_data
+
+    log_info "Script execution completed."
+}
+
+main

microservices/DSRInstrumentedCTORedmineGather/DSR-InstrumentedCTO-RedmineGather-12-20-2024-21-34-25-TestSuite.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# Copyright ReachableCEO Enterprises 2025
+# Licensed under the GNU Affero General Public License v3.0
+
+# Enable strict mode
+set -euo pipefail
+IFS=$'\n\t'
+
+# Test Suite for DSR-InstrumentedCTO-RedmineGather
+
+# Variables
+LOG_FILE="test-LOG-DSR-InstrumentedCTO-RedmineGather-12-20-2024-21-34-25.log"
+
+# Logging functions for tests
+log_info() {
+    local msg="$1"
+    echo -e "\033[32m[INFO] $(date '+%m-%d-%Y-%H-%M-%S') $msg\033[0m" | tee -a "$LOG_FILE"
+}
+
+log_error() {
+    local msg="$1"
+    echo -e "\033[31m[ERROR] $(date '+%m-%d-%Y-%H-%M-%S') $msg\033[0m" | tee -a "$LOG_FILE" >&2
+}
+
+# Test for gather_redmine_data
+test_gather_redmine_data() {
+    log_info "Testing gather_redmine_data function..."
+
+    # Mock Redmine API response
+    local mock_response='{"issues": []}'
+    echo "$mock_response" > redmine_data.json
+
+    if [[ -f redmine_data.json ]]; then
+        log_info "Test passed: Data file created."
+    else
+        log_error "Test failed: Data file not created."
+        exit 1
+    fi
+}
+
+# Run all tests
+main() {
+    log_info "Running tests for DSR-InstrumentedCTO-RedmineGather..."
+    test_gather_redmine_data
+    log_info "All tests passed."
+}
+
+main

vendor/git.knownelement.com/reachableceo/DSR-Pipeline-Server/build/build-stakeholder-output-server.sh

@@ -7,10 +7,14 @@ $MO_PATH $YamlInputTemplateFileStakeholderOutput > $BUILDYAML_STAKEHOLDER_OUTPUT
 
 echo "Creating stakeholder report..."
 
+cd "$(dirname $StakeholderOutputMarkdownInputFile)"
+
 pandoc \
 "$StakeholderOutputMarkdownOutputFile" \
 --template $PANDOC_TEMPLATE \
 --metadata-file="$BUILDYAML_STAKEHOLDER_OUTPUT" \
 --from markdown \
 --to=pdf \
---output $StakeholderOutputPDFOutputFile
+--output $StakeholderOutputPDFOutputFile
+
+cd -

vendor/git.knownelement.com/reachableceo/DSR-Pipeline-Server/build/publish-stakeholder-output-server.sh

@@ -1,5 +1,37 @@
 #!/bin/bash
 
+
+echo "Obtaining discourse api key..."
+
+bw logout
+
+####################################
+## Step 0: Set to use tsys server
+####################################
+bw config server $BITWARDEN_SERVER_URL
+
+####################################
+## Step 1: login to bitwarden
+####################################
+
+# From: https://bitwarden.com/help/cli/#using-an-api-key
+
+### Set apikey environment varaible
+
+source $BITWARDEN_CREDS
+
+### Login to vault using apikey...
+
+bw login --apikey $BW_CLIENTID $BW_CLIENTSECRET
+
+### Step 1.1: unlock / save session id 
+
+export BW_SESSION="$(bw unlock --passwordenv TSYS_BW_PASSWORD_REACHABLECEO --raw)"
+
+### Step 2: retrive a value into an environment variable
+
+export DISCOURSE_APIKEY="$(bw get password APIKEY-discourse)"
+
 echo "Posting Stakeholder Report..."
 
 # Check if the file exists
@@ -12,8 +44,8 @@ fi
 echo "Uploading file..."
 upload_response=$(curl -s -X POST "$DISCOURSE_URL/uploads.json" \
   -H "Content-Type: multipart/form-data" \
-  -H "Api-Key: $API_KEY" \
+  -H "Api-Key: $DISCOURSE_API_KEY" \
-  -H "Api-Username: $API_USERNAME" \
+  -H "Api-Username: $DISCOURSE_API_USERNAME" \
   -F "file=@$FILE_PATH;type=application/pdf" \
   -F "type=composer")
 
@@ -38,13 +70,13 @@ CONTENT="$CONTENT\n\n[Download todays report in PDF format]($short_url)"
 echo "Creating new topic..."
 post_response=$(curl -s -X POST "$DISCOURSE_URL/posts.json" \
   -H "Content-Type: application/json" \
-  -H "Api-Key: $API_KEY" \
+  -H "Api-Key: $DISCOURSE_API_KEY" \
-  -H "Api-Username: $API_USERNAME" \
+  -H "Api-Username: $DISCOURSE_API_USERNAME" \
   -d @- <<EOF
 {
-  "title": "$TITLE",
+  "title": "$DISCOURSE_POST_TITLE",
   "raw": "$CONTENT",
-  "category": $CATEGORY_ID
+  "category": $DISCOURSE_CATEGORY_ID
 }
 EOF
 )