feat(cloudron): add tirreno package artifacts

- Add CloudronStack/output/CloudronPackages-Artifacts/tirreno/ directory and its contents
- Includes package manifest, Dockerfile, source code, documentation, and build artifacts
- Add tirreno-1761840148.tar.gz as a build artifact
- Add tirreno-cloudron-package-1761841304.tar.gz as the Cloudron package
- Include all necessary files for the tirreno Cloudron package

This adds the complete tirreno Cloudron package artifacts to the repository.
This commit is contained in:
2025-10-30 11:43:06 -05:00
parent 0ce353ea9d
commit 91d52d2de5
1692 changed files with 202851 additions and 0 deletions

View File

@@ -0,0 +1,35 @@
<?php
/**
* Tirreno ~ Open source user analytics
* Copyright (c) Tirreno Technologies Sàrl (https://www.tirreno.com)
*
* Licensed under GNU Affero General Public License version 3 of the or any later version.
* For full copyright and license information, please see the LICENSE
* Redistributions of files must retain the above copyright notice.
*
* @copyright Copyright (c) Tirreno Technologies Sàrl (https://www.tirreno.com)
* @license https://opensource.org/licenses/AGPL-3.0 AGPL License
* @link https://www.tirreno.com Tirreno(tm)
*/
namespace Crons;
abstract class AbstractCron {
use \Traits\Db;
use \Traits\Debug;
protected $f3;
public function __construct() {
$this->f3 = \Base::instance();
$this->connectToDb(false);
}
protected function log(string $message): void {
$cronName = get_class($this);
$cronName = substr($cronName, strrpos($cronName, '\\') + 1);
echo sprintf('[%s] %s%s', $cronName, $message, PHP_EOL);
}
}

View File

@@ -0,0 +1,83 @@
<?php
/**
* Tirreno ~ Open source user analytics
* Copyright (c) Tirreno Technologies Sàrl (https://www.tirreno.com)
*
* Licensed under GNU Affero General Public License version 3 of the or any later version.
* For full copyright and license information, please see the LICENSE
* Redistributions of files must retain the above copyright notice.
*
* @copyright Copyright (c) Tirreno Technologies Sàrl (https://www.tirreno.com)
* @license https://opensource.org/licenses/AGPL-3.0 AGPL License
* @link https://www.tirreno.com Tirreno(tm)
*/
namespace Crons;
abstract class AbstractQueueCron extends AbstractCron {
protected \Models\Queue\AccountOperationQueue $accountOpQueueModel;
protected function processItems(): void {
$this->log('Start processing queue.');
$start = time();
$success = $failed = [];
$errors = [];
do {
$this->log(sprintf('Fetching next batch (%s) in queue.', \Utils\Variables::getAccountOperationQueueBatchSize()));
$items = $this->accountOpQueueModel->getNextBatchInQueue(\Utils\Variables::getAccountOperationQueueBatchSize());
if (!count($items)) {
break;
}
$items = \array_reverse($items); // to use array_pop later.
$this->accountOpQueueModel->setExecutingForBatch(\array_column($items, 'id'));
while (time() - $start < \Utils\Constants::get('ACCOUNT_OPERATION_QUEUE_EXECUTE_TIME_SEC')) {
$item = \array_pop($items); // array_pop has O(1) complexity, array_shift has O(n) complexity.
if (!$item) {
break;
}
try {
$this->processItem($item);
$success[] = $item;
} catch (\Throwable $e) {
$failed[] = $item;
$this->log(sprintf('Queue error %s.', $e->getMessage()));
$errors[] = sprintf('Error on %s: %s. Trace: %s', json_encode($item), $e->getMessage(), $e->getTraceAsString());
}
}
} while (time() - $start < \Utils\Constants::get('ACCOUNT_OPERATION_QUEUE_EXECUTE_TIME_SEC')); // allow another batch to be fetched if time permits.
$this->accountOpQueueModel->setCompletedForBatch(\array_column($success, 'id'));
$this->accountOpQueueModel->setFailedForBatch(\array_column($failed, 'id'));
$this->accountOpQueueModel->setWaitingForBatch(\array_column($items, 'id')); // unfinished items back to waiting.
if (count($errors)) {
$errObj = [
'code' => 500,
'message' => sprintf('Cron %s err', get_class($this)),
//'trace' => implode('; ', $errors),
'trace' => $errors[0],
'sql_log' => '',
];
\Utils\ErrorHandler::saveErrorInformation($this->f3, $errObj);
}
$this->log(sprintf(
'Processed %s items in %s seconds. %s items failed. %s items put back in queue.',
count($success),
time() - $start,
count($failed),
count($items),
));
}
abstract protected function processItem(array $item): void;
}

View File

@@ -0,0 +1,63 @@
<?php
/**
* Tirreno ~ Open source user analytics
* Copyright (c) Tirreno Technologies Sàrl (https://www.tirreno.com)
*
* Licensed under GNU Affero General Public License version 3 of the or any later version.
* For full copyright and license information, please see the LICENSE
* Redistributions of files must retain the above copyright notice.
*
* @copyright Copyright (c) Tirreno Technologies Sàrl (https://www.tirreno.com)
* @license https://opensource.org/licenses/AGPL-3.0 AGPL License
* @link https://www.tirreno.com Tirreno(tm)
*/
namespace Crons;
class BatchedNewEvents extends AbstractCron {
private \Models\Queue\QueueNewEventsCursor $cursorModel;
private \Models\Queue\AccountOperationQueue $accountOpQueueModel;
private \Models\Events $eventsModel;
public function __construct() {
parent::__construct();
$this->cursorModel = new \Models\Queue\QueueNewEventsCursor();
$this->eventsModel = new \Models\Events();
$actionType = new \Type\QueueAccountOperationActionType(\Type\QueueAccountOperationActionType::CALCULATE_RISK_SCORE);
$this->accountOpQueueModel = new \Models\Queue\AccountOperationQueue($actionType);
}
public function gatherNewEventsBatch(): void {
if (!$this->cursorModel->acquireLock() && !$this->cursorModel->unclog()) {
$this->log('Could not acquire the lock; another cron is probably already working on recently added events.');
return;
}
try {
$cursor = $this->cursorModel->getCursor();
$next = $this->cursorModel->getNextCursor($cursor, \Utils\Variables::getNewEventsBatchSize());
if ($next) {
$accounts = $this->eventsModel->getDistinctAccounts($cursor, $next);
$this->accountOpQueueModel->addBatch($accounts);
$this->cursorModel->updateCursor($next);
// Log new events cursor to database.
\Utils\Logger::log('Updated \'last_event_id\' in \'queue_new_events_cursor\' table to ', $next);
$this->log(sprintf('Added %s accounts to the risk score queue.', count($accounts)));
} else {
$this->log('No new events.');
}
} catch (\Throwable $e) {
$this->log(sprintf('Batched new events error %s.', $e->getMessage()));
} finally {
$this->cursorModel->releaseLock();
}
}
}

View File

@@ -0,0 +1,111 @@
<?php
/**
* Tirreno ~ Open source user analytics
* Copyright (c) Tirreno Technologies Sàrl (https://www.tirreno.com)
*
* Licensed under GNU Affero General Public License version 3 of the or any later version.
* For full copyright and license information, please see the LICENSE
* Redistributions of files must retain the above copyright notice.
*
* @copyright Copyright (c) Tirreno Technologies Sàrl (https://www.tirreno.com)
* @license https://opensource.org/licenses/AGPL-3.0 AGPL License
* @link https://www.tirreno.com Tirreno(tm)
*/
namespace Crons;
class BlacklistQueueHandler extends AbstractQueueCron {
public function __construct() {
parent::__construct();
$actionType = new \Type\QueueAccountOperationActionType(\Type\QueueAccountOperationActionType::BLACKLIST);
$this->accountOpQueueModel = new \Models\Queue\AccountOperationQueue($actionType);
}
public function processQueue(): void {
if ($this->accountOpQueueModel->isExecuting() && !$this->accountOpQueueModel->unclog()) {
$this->log('Blacklist queue is already being executed by another cron job.');
return;
}
$this->processItems($this->accountOpQueueModel);
}
protected function processItem(array $item): void {
$fraud = true;
$dataController = new \Controllers\Admin\User\Data();
$items = $dataController->setFraudFlag(
$item['event_account'],
$fraud,
$item['key'],
);
$model = new \Models\User();
$username = $model->getUser($item['event_account'], $item['key'])['userid'] ?? '';
$logger = new \Log('blacklist.log');
$logger->write('[BlacklistQueue] ' . $username . ' added to blacklist.');
$model = new \Models\ApiKeys();
$model->getKeyById($item['key']);
if (!$model->skip_blacklist_sync && $model->token) {
$user = new \Models\User();
$userEmail = $user->getUser($item['event_account'], $item['key'])['email'] ?? null;
if ($userEmail !== null) {
$hashes = $this->getHashes($items, $userEmail);
$errorMessage = $this->sendBlacklistReportPostRequest($hashes, $model->token);
if (strlen($errorMessage) > 0) {
// Log error to database
\Utils\Logger::log('Fraud enrichment API curl error', $errorMessage);
$this->log('Fraud enrichment API curl error logged to database.');
}
}
}
}
/**
* @param array<array{type: string, value: string}> $items
*/
private function getHashes(array $items, string $userEmail): array {
$userHash = hash('sha256', $userEmail);
return array_map(function ($item) use ($userHash) {
return [
'type' => $item['type'],
'value' => hash('sha256', $item['value']),
'id' => $userHash,
];
}, $items);
}
/**
* @param array<array{type: string, value: string}> $hashes
*/
private function sendBlacklistReportPostRequest(array $hashes, string $enrichmentKey): string {
$postFields = [
'data' => $hashes,
];
$options = [
'method' => 'POST',
'header' => [
'Content-Type: application/json',
'Authorization: Bearer ' . $enrichmentKey,
'User-Agent: ' . $this->f3->get('USER_AGENT'),
],
'content' => \json_encode($postFields),
];
/** @var array{request: array<string>, body: string, headers: array<string>, engine: string, cached: bool, error: string} $result */
$result = \Web::instance()->request(
url: \Utils\Variables::getEnrichtmentApi() . '/global_alert_report',
options: $options,
);
return $result['error'];
}
}

View File

@@ -0,0 +1,38 @@
<?php
/**
* Tirreno ~ Open source user analytics
* Copyright (c) Tirreno Technologies Sàrl (https://www.tirreno.com)
*
* Licensed under GNU Affero General Public License version 3 of the or any later version.
* For full copyright and license information, please see the LICENSE
* Redistributions of files must retain the above copyright notice.
*
* @copyright Copyright (c) Tirreno Technologies Sàrl (https://www.tirreno.com)
* @license https://opensource.org/licenses/AGPL-3.0 AGPL License
* @link https://www.tirreno.com Tirreno(tm)
*/
namespace Crons;
class DeletionQueueHandler extends AbstractQueueCron {
public function __construct() {
parent::__construct();
$actionType = new \Type\QueueAccountOperationActionType(\Type\QueueAccountOperationActionType::DELETE);
$this->accountOpQueueModel = new \Models\Queue\AccountOperationQueue($actionType);
}
public function processQueue(): void {
if ($this->accountOpQueueModel->isExecuting() && !$this->accountOpQueueModel->unclog()) {
$this->log('Deletion queue is already being executed by another cron job.');
} else {
$this->processItems($this->accountOpQueueModel);
}
}
protected function processItem(array $item): void {
$user = new \Models\User();
$user->deleteAllUserData($item['event_account'], $item['key']);
}
}

View File

@@ -0,0 +1,73 @@
<?php
/**
* Tirreno ~ Open source user analytics
* Copyright (c) Tirreno Technologies Sàrl (https://www.tirreno.com)
*
* Licensed under GNU Affero General Public License version 3 of the or any later version.
* For full copyright and license information, please see the LICENSE
* Redistributions of files must retain the above copyright notice.
*
* @copyright Copyright (c) Tirreno Technologies Sàrl (https://www.tirreno.com)
* @license https://opensource.org/licenses/AGPL-3.0 AGPL License
* @link https://www.tirreno.com Tirreno(tm)
*/
namespace Crons;
class EnrichmentQueueHandler extends AbstractQueueCron {
private \Models\ApiKeys $apiKeysModel;
private \Controllers\Admin\Enrichment\Data $controller;
public function __construct() {
parent::__construct();
$actionType = new \Type\QueueAccountOperationActionType(\Type\QueueAccountOperationActionType::ENRICHMENT);
$this->accountOpQueueModel = new \Models\Queue\AccountOperationQueue($actionType);
$this->apiKeysModel = new \Models\ApiKeys();
$this->controller = new \Controllers\Admin\Enrichment\Data();
}
public function processQueue(): void {
if ($this->accountOpQueueModel->isExecuting() && !$this->accountOpQueueModel->unclog()) {
$this->log('Enrchment queue is already being executed by another cron job.');
} else {
$this->processItems($this->accountOpQueueModel);
}
}
protected function processItem(array $item): void {
$start = time();
$apiKey = $item['key'];
$userId = $item['event_account'];
$subscriptionKey = $this->apiKeysModel->getKeyById($apiKey)->token;
$entities = $this->controller->getNotCheckedEntitiesByUserId($userId, $apiKey);
// TODO: check key ?
$this->log(sprintf('Items to enrich for account %s: %s.', $userId, json_encode($entities)));
$summary = [];
$success = 0;
$failed = 0;
foreach ($entities as $type => $items) {
if (count($items)) {
$summary[$type] = count($items);
}
foreach ($items as $item) {
$result = $this->controller->enrichEntity($type, null, $item, $apiKey, $subscriptionKey);
if (isset($result['ERROR_CODE'])) {
$failed += 1;
} else {
$success += 1;
}
}
}
// TODO: if failed !== 0 add to queue again?
// TODO: recalculate score after all?
$this->log(sprintf('Enrichment for account %s: %s enriched, %s failed in %s s (%s).', $userId, $success, $failed, time() - $start, json_encode($summary)));
}
}

View File

@@ -0,0 +1,35 @@
<?php
/**
* Tirreno ~ Open source user analytics
* Copyright (c) Tirreno Technologies Sàrl (https://www.tirreno.com)
*
* Licensed under GNU Affero General Public License version 3 of the or any later version.
* For full copyright and license information, please see the LICENSE
* Redistributions of files must retain the above copyright notice.
*
* @copyright Copyright (c) Tirreno Technologies Sàrl (https://www.tirreno.com)
* @license https://opensource.org/licenses/AGPL-3.0 AGPL License
* @link https://www.tirreno.com Tirreno(tm)
*/
namespace Crons;
class LogbookRotation extends AbstractCron {
public function rotateRequests(): void {
$this->log('Start logbook rotation.');
$model = new \Models\ApiKeys();
$keys = $model->getAllApiKeyIds();
// rotate events for unauthorized requests
$keys[] = ['id' => null];
$model = new \Models\Logbook();
$cnt = 0;
foreach ($keys as $key) {
$cnt += $model->rotateRequests($key['id']);
}
$this->log(sprintf('Deleted %s events for %s keys in logbook.', $cnt, count($keys)));
}
}

View File

@@ -0,0 +1,85 @@
<?php
/**
* Tirreno ~ Open source user analytics
* Copyright (c) Tirreno Technologies Sàrl (https://www.tirreno.com)
*
* Licensed under GNU Affero General Public License version 3 of the or any later version.
* For full copyright and license information, please see the LICENSE
* Redistributions of files must retain the above copyright notice.
*
* @copyright Copyright (c) Tirreno Technologies Sàrl (https://www.tirreno.com)
* @license https://opensource.org/licenses/AGPL-3.0 AGPL License
* @link https://www.tirreno.com Tirreno(tm)
*/
declare(strict_types=1);
namespace Crons;
class NotificationsHandler extends AbstractCron {
private const NOTIFICATION_WINDOW_HOUR_START = 9;
private const NOTIFICATION_WINDOW_HOUR_END = 17;
private \Models\NotificationPreferences $notificationPreferencesModel;
public function __construct() {
parent::__construct();
$this->notificationPreferencesModel = new \Models\NotificationPreferences();
}
public function prepareNotifications(): void {
$timezonesInWindow = $this->getTimeZonesInWindow(self::NOTIFICATION_WINDOW_HOUR_START, self::NOTIFICATION_WINDOW_HOUR_END);
$operatorsToNotify = $this->notificationPreferencesModel->listOperatorsEligableForUnreviewedItemsReminder($timezonesInWindow);
foreach ($operatorsToNotify as $operator) {
try {
$this->sendUnreviewedItemsReminderEmail($operator['firstname'] ?? '', $operator['email'], $operator['review_queue_cnt']);
} catch (\Throwable $e) {
$this->log(sprintf('Notification handler error %s.', $e->getMessage()));
}
}
$count = \count($operatorsToNotify);
if ($count > 0) {
$this->notificationPreferencesModel->updateLastUnreviewedItemsReminder(\array_column($operatorsToNotify, 'id'));
}
$this->log(sprintf('Sent %s unreviewed items reminder notifications.', $count));
}
/**
* @return string[] Time zones currently in the notification window
*/
private function getTimeZonesInWindow(int $startHour, int $endHour): array {
$timezones = \DateTimeZone::listIdentifiers();
return \array_filter($timezones, function ($timezone) use ($startHour, $endHour) {
$date = new \DateTime('now', new \DateTimeZone($timezone));
$hour = (int) $date->format('H');
return $hour >= $startHour && $hour < $endHour;
});
}
private function sendUnreviewedItemsReminderEmail(string $recipientFirstName, string $recipientEmail, int $reviewCount): void {
$audit = \Audit::instance();
if (!$audit->email($recipientEmail, true)) {
$this->log(sprintf('Username `%s` is not email; review count is %s', $recipientEmail, $reviewCount));
return;
}
$subject = $this->f3->get('UnreviewedItemsReminder_email_subject');
$subject = sprintf($subject, $reviewCount);
$message = $this->f3->get('UnreviewedItemsReminder_email_body');
$url = \Utils\Variables::getSiteWithProtocol();
$message = sprintf($message, $recipientFirstName, $recipientEmail, $reviewCount, $url);
\Utils\Mailer::send($recipientFirstName, $recipientEmail, $subject, $message);
}
}

View File

@@ -0,0 +1,38 @@
<?php
/**
* Tirreno ~ Open source user analytics
* Copyright (c) Tirreno Technologies Sàrl (https://www.tirreno.com)
*
* Licensed under GNU Affero General Public License version 3 of the or any later version.
* For full copyright and license information, please see the LICENSE
* Redistributions of files must retain the above copyright notice.
*
* @copyright Copyright (c) Tirreno Technologies Sàrl (https://www.tirreno.com)
* @license https://opensource.org/licenses/AGPL-3.0 AGPL License
* @link https://www.tirreno.com Tirreno(tm)
*/
namespace Crons;
class QueuesClearer extends AbstractCron {
public function clearQueues(): void {
$daysAgo = \Utils\Constants::get('ACCOUNT_OPERATION_QUEUE_CLEAR_COMPLETED_AFTER_DAYS');
$clearBefore = new \DateTime(sprintf('%s days ago', $daysAgo));
$actionTypes = [
new \Type\QueueAccountOperationActionType(\Type\QueueAccountOperationActionType::BLACKLIST),
new \Type\QueueAccountOperationActionType(\Type\QueueAccountOperationActionType::DELETE),
new \Type\QueueAccountOperationActionType(\Type\QueueAccountOperationActionType::CALCULATE_RISK_SCORE),
];
$clearedCount = 0;
foreach ($actionTypes as $type) {
$queue = new \Models\Queue\AccountOperationQueue($type);
$clearedCount += $queue->clearCompleted($clearBefore);
}
$this->log(sprintf('Cleared %s completed items.', $clearedCount));
}
}

View File

@@ -0,0 +1,37 @@
<?php
/**
* Tirreno ~ Open source user analytics
* Copyright (c) Tirreno Technologies Sàrl (https://www.tirreno.com)
*
* Licensed under GNU Affero General Public License version 3 of the or any later version.
* For full copyright and license information, please see the LICENSE
* Redistributions of files must retain the above copyright notice.
*
* @copyright Copyright (c) Tirreno Technologies Sàrl (https://www.tirreno.com)
* @license https://opensource.org/licenses/AGPL-3.0 AGPL License
* @link https://www.tirreno.com Tirreno(tm)
*/
namespace Crons;
class RetentionPolicyViolations extends AbstractCron {
public function gatherViolations(): void {
$this->log('Start retention policy violations.');
$eventsModel = new \Models\Events();
$retentionModel = new \Models\RetentionPolicies();
$retentionKeys = $retentionModel->getRetentionKeys();
$cnt = 0;
foreach ($retentionKeys as $key) {
// insuring clause
if ($key['retention_policy'] > 0) {
$cnt += $eventsModel->retentionDeletion($key['retention_policy'], $key['id']);
}
}
$this->log(sprintf('Deleted %s events for %s operators due to retention policy violations.', $cnt, count($retentionKeys)));
}
}

View File

@@ -0,0 +1,45 @@
<?php
/**
* Tirreno ~ Open source user analytics
* Copyright (c) Tirreno Technologies Sàrl (https://www.tirreno.com)
*
* Licensed under GNU Affero General Public License version 3 of the or any later version.
* For full copyright and license information, please see the LICENSE
* Redistributions of files must retain the above copyright notice.
*
* @copyright Copyright (c) Tirreno Technologies Sàrl (https://www.tirreno.com)
* @license https://opensource.org/licenses/AGPL-3.0 AGPL License
* @link https://www.tirreno.com Tirreno(tm)
*/
namespace Crons;
class RiskScoreQueueHandler extends AbstractQueueCron {
private \Models\OperatorsRules $rulesModel;
private \Controllers\Admin\Rules\Data $rulesController;
public function __construct() {
parent::__construct();
$actionType = new \Type\QueueAccountOperationActionType(\Type\QueueAccountOperationActionType::CALCULATE_RISK_SCORE);
$this->accountOpQueueModel = new \Models\Queue\AccountOperationQueue($actionType);
$this->rulesModel = new \Models\OperatorsRules();
$this->rulesController = new \Controllers\Admin\Rules\Data();
$this->rulesController->buildEvaluationModels();
}
public function processQueue(): void {
if ($this->accountOpQueueModel->isExecuting() && !$this->accountOpQueueModel->unclog()) {
$this->log('Risk score queue is already being executed by another cron job.');
return;
}
$this->processItems($this->accountOpQueueModel);
}
protected function processItem(array $item): void {
$this->rulesController->evaluateUser($item['event_account'], $item['key'], true);
}
}

View File

@@ -0,0 +1,51 @@
<?php
/**
* Tirreno ~ Open source user analytics
* Copyright (c) Tirreno Technologies Sàrl (https://www.tirreno.com)
*
* Licensed under GNU Affero General Public License version 3 of the or any later version.
* For full copyright and license information, please see the LICENSE
* Redistributions of files must retain the above copyright notice.
*
* @copyright Copyright (c) Tirreno Technologies Sàrl (https://www.tirreno.com)
* @license https://opensource.org/licenses/AGPL-3.0 AGPL License
* @link https://www.tirreno.com Tirreno(tm)
*/
namespace Crons;
class Totals extends AbstractCron {
// execute before risk score!
public function calculateTotals(): void {
$this->log('Start totals calculation.');
$start = time();
$models = \Utils\Constants::get('REST_TOTALS_MODELS');
$actionType = new \Type\QueueAccountOperationActionType(\Type\QueueAccountOperationActionType::CALCULATE_RISK_SCORE);
$queueModel = new \Models\Queue\AccountOperationQueue($actionType);
$keys = $queueModel->getNextBatchKeysInQueue(\Utils\Variables::getAccountOperationQueueBatchSize());
$res = [];
foreach ($models as $name => $modelClass) {
$res[$name] = ['cnt' => 0, 's' => 0];
$s = time();
$model = new $modelClass();
foreach ($keys as $key) {
(new \Models\SessionStat())->updateStats($key);
$cnt = $model->updateAllTotals($key);
$res[$name]['cnt'] += $cnt;
if (time() - $start > \Utils\Constants::get('ACCOUNT_OPERATION_QUEUE_EXECUTE_TIME_SEC')) {
// TODO: any reason to put the rest keys to queue?
$res[$name]['s'] = time() - $s;
break 2;
}
}
$res[$name]['s'] = time() - $s;
}
$this->log(sprintf('Updated %s entities for %s keys and %s models in %s seconds.', array_sum(array_column(array_values($res), 'cnt')), count($keys), count($models), time() - $start));
}
}

View File

@@ -0,0 +1,3 @@
<?php
//