diff --git a/app/SiteInfo.php b/app/SiteInfo.php index 5a7a12884..9c9214b5a 100644 --- a/app/SiteInfo.php +++ b/app/SiteInfo.php @@ -58,11 +58,29 @@ class SiteInfo extends Base { (string)shell_exec(BIN_PHINX . ' status -c ' . PHINX_MYSQL . ' --format=json|tail -n 1'), true )['migrations'], - fn($v) => count($v) > 0 + fn ($v) => count($v) > 0 ) ]; } + public function phinxMigrationsPending(): bool { + $pendingMigrations = false; + foreach ([PHINX_MYSQL, PHINX_MYSQL2, PHINX_PG] as $configuration) { + $output = shell_exec( + escapeshellarg(BIN_PHINX) . " status -c " . escapeshellarg($configuration) + . " --format=json | tail -n 1" + ); + if ($output !== false) { + $status = json_decode($output, true); + if (is_array($status) && $status['pending_count'] !== 0) { + $pendingMigrations = true; + break; + } + } + } + return $pendingMigrations; + } + public function composerVersion(): string { return trim((string)shell_exec(BIN_COMPOSER . ' --version 2>/dev/null')); } diff --git a/app/Task.php b/app/Task.php index 898b86207..e46470cec 100644 --- a/app/Task.php +++ b/app/Task.php @@ -20,60 +20,64 @@ abstract class Task extends Base { $this->startTime = hrtime(true); } - public function begin(): void { - self::$db->prepared_query(' - INSERT INTO periodic_task_history - (periodic_task_id) - VALUES (?) + public function begin(): int { + $this->historyId = $this->pg()->scalarex(' + insert into task_history + (id_task) + values ($1) + returning id_task_history ', $this->taskId ); - $this->historyId = self::$db->inserted_id(); + return $this->historyId; } public function end(bool $sane): int { $elapsed = (hrtime(true) - $this->startTime) / 1e6; $errorCount = count(array_filter($this->events, fn($event) => $event->severity === 'error')); - self::$db->prepared_query(' - UPDATE periodic_task_history SET - status = ?, - num_errors = ?, - num_items = ?, - duration_ms = ? - WHERE periodic_task_history_id = ? - ', 'completed', $errorCount, $this->processed, $elapsed, $this->historyId + $this->pg()->query(' + update task_history set + status = $1::task_status_t, + error_total = $2, + item_total = $3, + duration_ms = $4 + where id_task_history = $5 + ', 'completed', $errorCount, $this->processed, (int)$elapsed, $this->historyId ); echo("DONE! (" . number_format((hrtime(true) - $this->startTime) * 1e-9, 3) . ")\n"); foreach ($this->events as $event) { - printf("%s [%s] (%d) %s\n", $event->timestamp, $event->severity, $event->reference, $event->event); - self::$db->prepared_query(' - INSERT INTO periodic_task_history_event - (periodic_task_history_id, severity, event_time, event, reference) - VALUES (?, ?, ?, substr(?, 1, 255), ?) - ', $this->historyId, $event->severity, $event->timestamp, $event->event, $event->reference); + $timestamp = $event->timestamp()->format('Y-m-d H:i:s'); + printf("%s [%s] (%d) %s\n", $timestamp, $event->severity, $event->reference, $event->event); + $this->pg()->query(' + insert into task_history_event + (id_task_history, created, reference, severity, note) + VALUES ($1, $2, $3, $4::task_event_severity_t, substr($5, 1, 1000)) + ', $this->historyId, $timestamp, $event->reference, $event->severity, $event->event); } if ($errorCount > 0 && $sane) { - self::$db->prepared_query(' - UPDATE periodic_task SET - is_sane = FALSE - WHERE periodic_task_id = ? - ', $this->taskId); + $this->pg()->query(' + update task set + is_sane = false + where id_task = $1 + ', $this->taskId + ); self::$cache->delete_value(TaskScheduler::CACHE_TASKS); - Irc::sendMessage( IRC_CHAN_DEV, - "Task {$this->name} is no longer sane " . SITE_URL . "/tools.php?action=periodic&mode=detail&id={$this->taskId}" + "Task {$this->name} is no longer sane " + . SITE_URL + . "/tools.php?action=periodic&mode=detail&id={$this->taskId}" + ); + } elseif ($errorCount === 0 && !$sane) { + $this->pg()->query(' + update task set + is_sane = true + where id_task = $1 + ', $this->taskId ); - } elseif ($errorCount == 0 && !$sane) { - self::$db->prepared_query(' - UPDATE periodic_task SET - is_sane = TRUE - WHERE periodic_task_id = ? - ', $this->taskId); self::$cache->delete_value(TaskScheduler::CACHE_TASKS); - Irc::sendMessage(IRC_CHAN_DEV, "Task {$this->name} is now sane"); } return $this->processed; diff --git a/app/Task/DisableStuckTasks.php b/app/Task/DisableStuckTasks.php index e15083897..581ebcaaf 100644 --- a/app/Task/DisableStuckTasks.php +++ b/app/Task/DisableStuckTasks.php @@ -8,37 +8,40 @@ use Gazelle\Util\Time; class DisableStuckTasks extends \Gazelle\Task { public function run(): void { // If a task fails with a fatal error it will be stuck in a `running` state forever - self::$db->prepared_query(" - SELECT pth.periodic_task_id, pth.periodic_task_history_id, pth.launch_time, pt.name - FROM periodic_task_history pth - INNER JOIN periodic_task pt USING (periodic_task_id) - WHERE pth.status = 'running' - AND pth.launch_time < now() - INTERVAL 15 MINUTE - AND pt.is_enabled IS TRUE + $taskList = $this->pg()->allex(" + select th.id_task, th.id_task_history, th.created, t.name + from task_history th + inner join task t using (id_task) + where th.status = 'running' + and th.created < now() - interval '15 minute' + and t.is_enabled is true "); - $tasks = self::$db->to_array(false, MYSQLI_ASSOC); - foreach ($tasks as $task) { - [$id, $historyId, $launchTime, $name] = array_values($task); - $duration = Time::diff(time() - (int)strtotime($launchTime) + time(), 2, false); + $now = new \DateTimeImmutable(); + foreach ($taskList as $task) { + $duration = sprintf( + '%0.2fs', + $now->format('U.u') - $task['created']->format('U.u') + ); - Irc::sendMessage(IRC_CHAN_DEV, "Marking stuck task $name ($duration) as insane"); + Irc::sendMessage(IRC_CHAN_DEV, "Marking stuck task {$task['name']} ($duration) as insane"); $this->processed++; - $this->info("Marking stuck task $name ($duration) as insane", $id); + $this->info("Marking stuck task {$task['name']} ($duration) as insane", $task['id_task']); - self::$db->prepared_query(' - UPDATE periodic_task SET - is_sane = FALSE - WHERE periodic_task_id = ? - ', $id + $this->pg()->begin(); + $this->pg()->query(' + update task set + is_sane = false + where id_task = $1 + ', $task['id_task'] ); - - self::$db->prepared_query(" - UPDATE periodic_task_history SET + $this->pg()->query(" + update task_history set status = 'failed' - WHERE periodic_task_history_id = ? - ", $historyId + where id_task_history = $1 + ", $task['id_task_history'] ); + $this->pg()->commit(); } if ($this->processed > 0) { diff --git a/app/Task/PurgeOldTaskHistory.php b/app/Task/PurgeOldTaskHistory.php index 1982056d4..c6bc7eca5 100644 --- a/app/Task/PurgeOldTaskHistory.php +++ b/app/Task/PurgeOldTaskHistory.php @@ -4,10 +4,9 @@ namespace Gazelle\Task; class PurgeOldTaskHistory extends \Gazelle\Task { public function run(): void { - self::$db->prepared_query(' - DELETE FROM periodic_task_history - WHERE launch_time < now() - INTERVAL 6 MONTH - '); - $this->processed = self::$db->affected_rows(); + $this->processed = $this->pg()->query(" + delete from task_history + where created < now() - interval '6 month' + "); } } diff --git a/app/TaskScheduler.php b/app/TaskScheduler.php index 323f87bb3..cd641afd3 100644 --- a/app/TaskScheduler.php +++ b/app/TaskScheduler.php @@ -8,7 +8,7 @@ namespace Gazelle; */ class TaskScheduler extends Base { - final public const CACHE_TASKS = 'scheduled_tasks'; + final public const CACHE_TASKS = 'tasks'; protected Util\SortableTableHeader $heading; @@ -19,23 +19,102 @@ class TaskScheduler extends Base { public function findByName(string $className): ?array { return $this->findById( - (int)self::$db->scalar(" - SELECT pt.periodic_task_id FROM periodic_task pt WHERE pt.classname = ? + (int)$this->pg()->scalarex(" + select id_task from task where class_name = $1 ", $className ) ); } - public function taskList(): array { - self::$db->prepared_query(" - SELECT periodic_task_id, name, classname, description, period, is_enabled, is_sane, is_debug, run_now - FROM periodic_task - "); - return self::$db->to_array('periodic_task_id', MYSQLI_ASSOC); + public function heading(): Util\SortableTableHeader { + return $this->heading ??= new Util\SortableTableHeader( + 'next', [ + 'name' => ['dbColumn' => 'name', 'defaultSort' => 'asc', 'text' => 'Name'], + 'period' => ['dbColumn' => 'period', 'defaultSort' => 'asc', 'text' => 'Interval'], + 'run_total' => ['dbColumn' => 'run_total', 'defaultSort' => 'desc', 'text' => 'Runs'], + 'duration' => ['dbColumn' => 'duration', 'defaultSort' => 'desc', 'text' => 'Duration'], + 'item_total' => ['dbColumn' => 'item_total', 'defaultSort' => 'desc', 'text' => 'Processed'], + 'status' => ['dbColumn' => 'status', 'defaultSort' => 'desc', 'text' => 'Status'], + 'error_total' => ['dbColumn' => 'error_total', 'defaultSort' => 'desc', 'text' => 'Errors'], + 'event_total' => ['dbColumn' => 'event_total', 'defaultSort' => 'desc', 'text' => 'Events'], + 'last' => [ + 'dbColumn' => 'last_run is null asc, is_enabled desc, last_run', + 'defaultSort' => 'desc', + 'text' => 'Last Run' + ], + 'next' => [ + 'dbColumn' => 'next_run is null asc, is_enabled desc, run_now desc, next_run', + 'defaultSort' => 'asc', + 'text' => 'Next Run' + ], + ] + ); } - public function insaneTaskList(): int { - return count(array_filter($this->taskList(), + public function taskDetailList(int $days = 7): array { + return $this->pg()->allex(<< now() - interval '1 day' * $1 + group by th.id_task + ), + stats(id_task, id_task_history, run_total, error_total, item_total) as ( + select id_task, + max(id_task_history), + count(*), + sum(error_total), + sum(item_total) + from task_history + where created > now() - interval '1 day' * $1 + group by id_task + ) + select * from ( + select t.id_task, t.name, t.description, t.period, t.is_enabled, t.is_sane, t.run_now, + coalesce(s.run_total, 0) as run_total, + coalesce(s.item_total, 0) as item_total, + coalesce(s.error_total, 0) as error_total, + coalesce(e.total, 0) as event_total, + coalesce(th.duration_ms, 0) as duration, + th.status, + th.created as last_run, + th.created + interval '1 second' * t.period + as next_run + from task t + left join events e on (e.id_task = t.id_task) + left join stats s on (s.id_task = t.id_task) + left join task_history th using (id_task_history) + ) q + order by {$this->heading()->orderBy()} {$this->heading()->dir()}, name asc + END_SQL, $days + ); + // need to wrap the select so that pg can resolve next_run + } + + public function taskList(): array { + return $this->pg()->allByKeyex( + 'id_task', + <<taskList(), fn ($v) => !$v['is_sane'] )); } @@ -49,54 +128,47 @@ class TaskScheduler extends Base { } public function enqueue(int $taskId): int { - self::$db->prepared_query(" - UPDATE periodic_task SET + return $this->pg()->query(' + update task set is_sane = true, run_now = true - WHERE periodic_task_id = ? - ", $taskId + where id_task = $1 + ', $taskId ); - return self::$db->affected_rows(); } public function clear(int $taskId): int { - self::$db->prepared_query(' - UPDATE periodic_task SET + return $this->pg()->query(' + update task set run_now = false - WHERE periodic_task_id = ? + where id_task = $1 ', $taskId ); - return self::$db->affected_rows(); } public function updateTask( int $taskId, string $name, - string $class, string $description, int $period, bool $isEnabled, bool $isSane, bool $isDebug ): int { - if (!$this->isClassValid($class)) { - return 0; - } - self::$db->prepared_query(" - UPDATE periodic_task SET - name = ?, - classname = ?, - description = ?, - period = ?, - is_enabled = ?, - is_sane = ?, - is_debug = ? - WHERE periodic_task_id = ? - ", $name, $class, $description, $period, - (int)$isEnabled, (int)$isSane, (int)$isDebug, + $affected = $this->pg()->query(' + update task set + name = $1, + description = $2, + period = $3, + is_enabled = $4, + is_sane = $5, + is_debug = $6 + where id_task = $7 + ', $name, $description, $period, $isEnabled, $isSane, $isDebug, $taskId, ); - return self::$db->affected_rows(); + self::$cache->delete_value(self::CACHE_TASKS); + return $affected; } public function runClass(string $className, bool $debug = false): int { @@ -104,29 +176,20 @@ class TaskScheduler extends Base { if ($task === null) { return -1; } - return $this->runTask($task['periodic_task_id'], $debug); + return $this->runTask($task['id_task'], $debug); } public function runNow(int $taskId): int { - self::$db->prepared_query(" - UPDATE periodic_task SET - run_now = 1 - run_now - WHERE periodic_task_id = ? + return $this->pg()->query(" + update task set + run_now = true + where id_task = $1 ", $taskId ); - return self::$db->affected_rows(); } public function run(): int { - $pendingMigrations = array_filter( - json_decode( - (string)shell_exec(BIN_PHINX . ' status -c ' . PHINX_MYSQL . ' --format=json | tail -n 1'), - true - )['migrations'], - fn ($value) => count($value) > 0 && $value['migration_status'] === 'down' - ); - - if ($pendingMigrations) { + if (new SiteInfo()->phinxMigrationsPending()) { Util\Irc::sendMessage(IRC_CHAN_DEV, 'Pending migrations found, scheduler cannot continue'); echo "Pending migrations found, aborting\n"; return 0; @@ -145,31 +208,31 @@ class TaskScheduler extends Base { $TTL = hrtime(true) + 58e9; while (hrtime(true) < $TTL) { - $taskId = (int)self::$db->scalar(" - SELECT pt.periodic_task_id - FROM periodic_task pt - LEFT JOIN ( - SELECT pth.periodic_task_id, - max(pth.launch_time) AS launch_time - FROM periodic_task_history pth - WHERE pth.status = 'completed' - GROUP BY pth.periodic_task_id - ) last USING (periodic_task_id) - WHERE pt.is_enabled IS TRUE - AND pt.is_sane IS TRUE - AND ( - last.periodic_task_id is null - OR last.launch_time + INTERVAL pt.period SECOND < now() - OR pt.run_now IS TRUE + $taskId = (int)$this->pg()->scalarex(" + select t.id_task + from task t + left join ( + select th.id_task, + max(th.created) AS created + from task_history th + where th.status = 'completed' + group by th.id_task + ) last using (id_task) + where t.is_enabled is true + and t.is_sane is true + and ( + last.id_task is null + or last.created + interval '1 second' * t.period < now() + or t.run_now is true ) - AND NOT EXISTS ( - SELECT 1 FROM periodic_task_history r - WHERE r.status = 'running' - AND r.periodic_task_id = pt.periodic_task_id + and not exists ( + select 1 from task_history r + where r.status = 'running' + anD r.id_task = t.id_task ) - AND pt.periodic_task_id NOT IN (" . placeholders($fail) . ") - LIMIT 1 - ", ...$fail + and not t.id_task = any($1::integer[]) + limit 1 + ", $this->pg()->integerConverter()->output($fail) ); if (!$taskId) { // no tasks remaining to be run @@ -191,10 +254,19 @@ class TaskScheduler extends Base { } echo "Running task {$task['name']}..."; - $taskRunner = $this->createRunner($taskId, $task['name'], $task['classname'], $task['is_debug'] || $debug); + $taskRunner = $this->createRunner($taskId, $task['name'], $task['class_name'], $task['is_debug'] || $debug); if (is_null($taskRunner)) { - echo "DONE! (0.000)\n"; - Util\Irc::sendMessage(IRC_CHAN_DEV, "Failed to construct task {$task['name']}"); + echo "Failed.\n"; + Util\Irc::sendMessage( + IRC_CHAN_DEV, + sprintf("No implementation for task $taskId (%s), disabling.", self::className($task['class_name'])) + ); + $this->pg()->query(' + update task set + is_enabled = false + where id_task = $1 + ', $taskId + ); return -1; } @@ -237,102 +309,37 @@ class TaskScheduler extends Base { } public function taskRunTotal(int $taskId): int { - return (int)self::$db->scalar(" - SELECT count(*) FROM periodic_task_history WHERE periodic_task_id = ? + return (int)$this->pg()->scalarex(" + select count(*) from task_history where id_task = $1 ", $taskId ); } - public function heading(): Util\SortableTableHeader { - return $this->heading ??= new Util\SortableTableHeader( - 'next', [ - 'name' => ['dbColumn' => 'name', 'defaultSort' => 'asc', 'text' => 'Name'], - 'period' => ['dbColumn' => 'period', 'defaultSort' => 'asc', 'text' => 'Interval'], - 'runs' => ['dbColumn' => 'runs', 'defaultSort' => 'desc', 'text' => 'Runs'], - 'duration' => ['dbColumn' => 'duration', 'defaultSort' => 'desc', 'text' => 'Duration'], - 'processed' => ['dbColumn' => 'processed', 'defaultSort' => 'desc', 'text' => 'Processed'], - 'status' => ['dbColumn' => 'status', 'defaultSort' => 'desc', 'text' => 'Status'], - 'errors' => ['dbColumn' => 'errors', 'defaultSort' => 'desc', 'text' => 'Errors'], - 'events' => ['dbColumn' => 'events', 'defaultSort' => 'desc', 'text' => 'Events'], - 'last' => [ - 'dbColumn' => 'last_run IS NULL ASC, is_enabled DESC, last_run', - 'defaultSort' => 'desc', - 'text' => 'Last Run' - ], - 'next' => [ - 'dbColumn' => 'next_run IS NULL ASC, is_enabled DESC, run_now DESC, next_run', - 'defaultSort' => 'asc', - 'text' => 'Next Run' - ], - ] - ); - } - - public function taskDetailList(int $days = 7): array { - self::$db->prepared_query(" - SELECT pt.periodic_task_id, - pt.period, - pt.name, - pt.description, - pt.period, - pt.is_enabled, - pt.is_sane, - pt.run_now, - last_value(pth.status) OVER (PARTITION BY pt.periodic_task_id, pt.period) - AS status, - count(DISTINCT pth.periodic_task_history_id) AS runs, - sum(DISTINCT pth.num_items) AS processed, - sum(DISTINCT pth.num_errors) AS errors, - sum(DISTINCT pth.duration_ms) AS duration, - count(pthe.periodic_task_history_event_id) AS events, - max(pth.launch_time) AS last_run, - max(pth.launch_time) + INTERVAL period SECOND AS next_run - FROM periodic_task pt - LEFT JOIN periodic_task_history pth USING (periodic_task_id) - LEFT JOIN periodic_task_history_event pthe USING (periodic_task_history_id) - WHERE ( - pth.launch_time IS NULL - OR pth.launch_time > now() - INTERVAL ? DAY - ) - GROUP BY pt.periodic_task_id, - pt.period, - pt.name, - pt.description, - pt.period, - pt.is_enabled, - pt.is_sane, - pt.run_now - ORDER BY {$this->heading()->orderBy()} {$this->heading()->dir()}, name ASC - ", $days - ); - return self::$db->to_array(false, MYSQLI_ASSOC); - } - public function taskHistory( int $taskId, int $limit, int $offset, ): ?TaskScheduler\TaskHistory { - self::$db->prepared_query(" - SELECT periodic_task_history_id, launch_time, status, num_errors, num_items, duration_ms - FROM periodic_task_history - WHERE periodic_task_id = ? - ORDER BY launch_time DESC - LIMIT ? OFFSET ? - ", $taskId, $limit, $offset + $items = $this->pg()->allex(' + select id_task_history, created, status, error_total, item_total, duration_ms + from task_history + where id_task = $1 + order by created desc + limit $2 offset $3 + ', $taskId, $limit, $offset ); - $items = self::$db->to_array('periodic_task_history_id', MYSQLI_ASSOC); $historyEvents = []; - if (count($items)) { - self::$db->prepared_query(" - SELECT periodic_task_history_id, event_time, severity, event, reference - FROM periodic_task_history_event - WHERE periodic_task_history_id IN (" . placeholders($items) . ") - ORDER BY event_time, periodic_task_history_event_id - ", ...array_keys($items)); - $events = self::$db->to_array(false, MYSQLI_ASSOC); - + if ($items !== []) { + $events = $this->pg()->allex(' + select id_task_history, created, severity, note, reference + from task_history_event + where id_task_history = any($1::integer[]) + order by created, id_task_history_event + ', $this->pg()->integerConverter()->output( + array_column($items, 'id_task_history') + ) + ); foreach ($events as $event) { [$historyId, $eventTime, $severity, $message, $reference] = array_values($event); $historyEvents[$historyId][] = new TaskScheduler\Event($severity, $message, $reference, $eventTime); @@ -344,115 +351,120 @@ class TaskScheduler extends Base { $this->taskRunTotal($taskId) ); foreach ($items as $item) { - [$historyId, $launchTime, $status, $numErrors, $numItems, $duration] = array_values($item); + [$historyId, $created, $status, $errorTotal, $itemTotal, $duration] = array_values($item); $taskEvents = $historyEvents[$historyId] ?? []; - $history->items[] = new TaskScheduler\HistoryItem($launchTime, $status, $numErrors, $numItems, $duration, $taskEvents); + $history->items[] = new TaskScheduler\HistoryItem($created, $status, $errorTotal, $itemTotal, $duration, $taskEvents); } return $history; } - private function constructAxes(array $data, string $key, array $axes, bool $time): array { - $result = []; - - foreach ($axes as $axis) { - if (is_array($axis)) { - $taskId = $axis[0]; - $name = $axis[1]; - } else { - $taskId = $axis; - $name = $axis; - } - - $result[] = [ - 'name' => $name, - 'data' => array_map( - fn ($v) => [$time ? (int)strtotime($v[$key]) * 1000 : $v[$key], (int)$v[$taskId]], - $data - ) - ]; - } - return $result; - } - public function runtimeStats(int $days = 90): array { - self::$db->prepared_query(" - SELECT date_format(pth.launch_time, '%Y-%m-%d %H:00:00') AS date, - sum(pth.duration_ms) AS duration, - sum(pth.num_items) AS processed - FROM periodic_task pt - INNER JOIN periodic_task_history pth USING (periodic_task_id) - WHERE pt.is_enabled IS TRUE - AND pth.launch_time >= now() - INTERVAL 1 DAY - GROUP BY 1 - ORDER BY 1 - "); - $hourly = $this->constructAxes(self::$db->to_array(false, MYSQLI_ASSOC), 'date', ['duration', 'processed'], true); - - self::$db->prepared_query(" - SELECT date(pth.launch_time) AS date, - sum(pth.duration_ms) AS duration, - sum(pth.num_items) AS processed - FROM periodic_task pt - INNER JOIN periodic_task_history pth USING (periodic_task_id) - WHERE pt.is_enabled IS TRUE - AND pth.launch_time BETWEEN curdate() - INTERVAL ? DAY AND curdate() - GROUP BY 1 - ORDER BY 1 - ", $days - ); - $daily = $this->constructAxes(self::$db->to_array(false, MYSQLI_ASSOC), 'date', ['duration', 'processed'], true); - - self::$db->prepared_query(" - SELECT pt.name, - avg(pth.duration_ms) AS duration_avg, - avg(pth.num_items) AS processed_avg - FROM periodic_task pt - INNER JOIN periodic_task_history pth USING (periodic_task_id) - WHERE pt.is_enabled IS TRUE - AND pth.launch_time BETWEEN curdate() - INTERVAL ? DAY AND curdate() - GROUP BY 1 - ORDER BY 1 - ", $days - ); - $tasks = $this->constructAxes(self::$db->to_array(false, MYSQLI_ASSOC), 'name', ['duration_avg', 'processed_avg'], false); - - $totals = self::$db->rowAssoc(" - SELECT count(pth.periodic_task_history_id) AS runs, - sum(pth.duration_ms) AS duration, - sum(pth.num_items) AS processed, - count(pthe.periodic_task_history_event_id) AS events, - sum(pth.num_errors) AS errors - FROM periodic_task pt - INNER JOIN periodic_task_history pth USING (periodic_task_id) - LEFT JOIN periodic_task_history_event pthe USING (periodic_task_history_id) - WHERE pt.is_enabled IS TRUE - AND pth.launch_time BETWEEN curdate() - INTERVAL ? DAY AND curdate() - ", $days + $daily = $this->pg()->rowAssocex(<< $hourly, - 'daily' => $daily, - 'tasks' => $tasks, - 'totals' => $totals, + 'hourly' => $this->pg()->rowAssocex(<< now() - interval '1 day' + group by h + order by h + ) + select + jsonb_agg(h) as category, + jsonb_agg(duration) as duration, + jsonb_agg(item_total) as item_total + from hourly + END_SQL + ), + 'daily' => [ + 'category' => $daily['daily_category'], + 'duration' => $daily['daily_duration'], + 'item_total' => $daily['daily_total'], + ], + 'tasks' => [ + 'category' => $daily['task_category'], + 'duration' => $daily['task_duration'], + 'item_total' => $daily['task_total'], + ], + 'totals' => $this->pg()->rowAssocex(<<prepared_query(" - SELECT date(pth.launch_time) AS date, - sum(pth.duration_ms) AS duration, - sum(pth.num_items) AS processed - FROM periodic_task pt - INNER JOIN periodic_task_history pth USING (periodic_task_id) - WHERE pt.periodic_task_id = ? - AND pth.launch_time BETWEEN curdate() - INTERVAL ? DAY AND curdate() - GROUP BY 1 - ORDER BY 1 - ", $taskId, $days + return $this->pg()->rowAssocex(<<constructAxes(self::$db->to_array(false, MYSQLI_ASSOC), 'date', ['duration', 'processed'], true); + public function isLastRunIntervalOk(int $delay): bool { + return $delay >= (int)$this->pg()->scalarex(" + select extract(epoch from now() - max(created)) from task_history + "); } } diff --git a/app/TaskScheduler/Event.php b/app/TaskScheduler/Event.php index 9f904888a..d21db6f40 100644 --- a/app/TaskScheduler/Event.php +++ b/app/TaskScheduler/Event.php @@ -3,14 +3,18 @@ namespace Gazelle\TaskScheduler; class Event { - public string $timestamp; - public function __construct( - public readonly string $severity, - public readonly string $event, - public readonly int $reference, - string|null $timestamp = null, + public readonly string $severity, + public readonly string $event, + public readonly int $reference, + protected \DateTimeImmutable|null $timestamp = null, ) { - $this->timestamp = $timestamp ?? date('Y-m-d H:i:s'); + if (is_null($this->timestamp)) { + $this->timestamp = new \DateTimeImmutable(); + } + } + + public function timestamp(): \DateTimeImmutable { + return $this->timestamp; } } diff --git a/app/TaskScheduler/HistoryItem.php b/app/TaskScheduler/HistoryItem.php index 5a6d1ad95..6a94aa99e 100644 --- a/app/TaskScheduler/HistoryItem.php +++ b/app/TaskScheduler/HistoryItem.php @@ -4,11 +4,11 @@ namespace Gazelle\TaskScheduler; class HistoryItem { public function __construct( - public readonly string $launchTime, - public readonly string $status, - public readonly int $numErrors, - public readonly int $numItems, - public readonly int $duration, - public readonly array $events = [], + public readonly \DateTimeImmutable $created, + public readonly string $status, + public readonly int $errorTotal, + public readonly int $itemTotal, + public readonly int $duration, + public readonly array $events = [], ) {} } diff --git a/app/User/Activity.php b/app/User/Activity.php index 1eb5b2b2c..304f1ba99 100644 --- a/app/User/Activity.php +++ b/app/User/Activity.php @@ -116,13 +116,10 @@ class Activity extends \Gazelle\BaseUser { public function setScheduler(\Gazelle\TaskScheduler $scheduler): static { if ($this->user->permitted('admin_periodic_task_view')) { - $lastSchedulerRun = self::$db->scalar(" - SELECT now() - max(launch_time) FROM periodic_task_history - "); - if ($lastSchedulerRun > SCHEDULER_DELAY) { + if (!$scheduler->isLastRunIntervalOk(SCHEDULER_DELAY)) { $this->setAlert("CRON"); } - $insane = $scheduler->insaneTaskList(); + $insane = $scheduler->insaneTaskTotal(); if ($insane) { $this->setAlert("table('task', ['id' => false, 'primary_key' => 'id_task']) + ->addColumn('id_task', 'integer', ['identity' => true]) + ->addColumn('period', 'integer') + ->addColumn('is_enabled', 'boolean', ['default' => true]) + ->addColumn('is_sane', 'boolean', ['default' => true]) + ->addColumn('is_debug', 'boolean', ['default' => false]) + ->addColumn('run_now', 'boolean', ['default' => false]) + ->addColumn('name', 'string', ['length' => 64]) + ->addColumn('class_name', 'string', ['length' => 32]) + ->addColumn('description', 'string', ['length' => 100]) + ->addIndex(['name'], ['unique' => true, 'name' => 'pt_n_uidx']) + ->addIndex(['class_name'], ['unique' => true, 'name' => 'pt_cn_uidx']) + ->save(); + + $this->query(" + create type task_status_t as enum ('running', 'completed', 'failed') + "); + + $this->table('task_history', ['id' => false, 'primary_key' => 'id_task_history']) + ->addColumn('id_task_history', 'integer', ['identity' => true]) + ->addColumn('id_task', 'integer') + ->addColumn('error_total', 'integer', ['default' => 0]) + ->addColumn('item_total', 'integer', ['default' => 0]) + ->addColumn('duration_ms', 'integer', ['default' => 0]) + ->addColumn('created', 'timestamp', ['timezone' => true, 'default' => 'CURRENT_TIMESTAMP']) + ->addColumn('status', Literal::from('task_status_t'), ['default' => 'running']) + ->addIndex(['created'], ['name' => 'pth_c_idx']) + ->addIndex(['id_task'], ['name' => 'th_t_idx']) + ->addForeignKey('id_task', 'task', 'id_task', ['delete' => 'CASCADE', 'update' => 'CASCADE']) + ->save(); + + $this->query(" + create type task_event_severity_t as enum ('error', 'info', 'debug') + "); + + $this->table('task_history_event', ['id' => false, 'primary_key' => 'id_task_history_event']) + ->addColumn('id_task_history_event', 'integer', ['identity' => true]) + ->addColumn('id_task_history', 'integer') + ->addColumn('reference', 'integer') + ->addColumn('created', 'timestamp', ['timezone' => true, 'default' => 'CURRENT_TIMESTAMP']) + ->addColumn('severity', Literal::from('task_event_severity_t')) + ->addColumn('note', 'string', ['length' => 1000]) + ->addIndex(['id_task_history'], ['name' => 'the_th_idx']) + ->addForeignKey('id_task_history', 'task_history', 'id_task_history', ['delete' => 'CASCADE', 'update' => 'CASCADE']) + ->save(); + + $this->query(' + insert into task (id_task, period, is_enabled, is_sane, is_debug, run_now, name, class_name, description) + select periodic_task_id, + period, + case when is_enabled = 0 then false else true end, + case when is_sane = 0 then false else true end, + case when is_debug = 0 then false else true end, + case when run_now = 0 then false else true end, + name, + classname, + description + from relay.periodic_task + '); + $this->query(<<query(" + select + setval( + 'task_id_task_seq', + (select max(id_task) from task) + ), + setval( + 'task_history_id_task_history_seq', + (select max(id_task_history) from task_history) + ) + "); + } + + public function down(): void { + $this->table('task_history_event')->drop()->save(); + $this->table('task_history')->drop()->save(); + $this->table('task')->drop()->save(); + + $this->query(" + drop type task_event_severity_t + "); + $this->query(" + drop type task_status_t + "); + } +} diff --git a/sections/tools/development/periodic_alter.php b/sections/tools/development/periodic_alter.php index 6b1f464cb..a50dcfc0f 100644 --- a/sections/tools/development/periodic_alter.php +++ b/sections/tools/development/periodic_alter.php @@ -15,7 +15,6 @@ if ($taskId && $_POST['submit'] == 'Edit') { $validator = new Util\Validator(); $validator->setFields([ ['name', true, 'string', 'The name must be set, and has a max length of 64 characters', ['maxlength' => 64]], - ['classname', true, 'string', 'The class name must be set, and has a max length of 32 characters', ['maxlength' => 32]], ['description', true, 'string', 'The description must be set, and has a max length of 255 characters', ['maxlength' => 255]], ['interval', true, 'number', 'The interval must be a number'], ]); @@ -26,10 +25,10 @@ if ($taskId && $_POST['submit'] == 'Edit') { if ($task == null) { $err = "Task $taskId not found"; } - $scheduler->updateTask($taskId, $_POST['name'], $_POST['classname'], $_POST['description'], + $scheduler->updateTask($taskId, $_POST['name'], $_POST['description'], (int)$_POST['interval'], isset($_POST['enabled']), isset($_POST['sane']), isset($_POST['debug']) ); } } -header('Location: tools.php?action=periodic&mode=edit'); +header('Location: ?action=periodic&mode=edit'); diff --git a/sections/tools/development/periodic_detail.php b/sections/tools/development/periodic_detail.php index a73b04d56..04b9a49b5 100644 --- a/sections/tools/development/periodic_detail.php +++ b/sections/tools/development/periodic_detail.php @@ -18,14 +18,11 @@ if (!$scheduler->findById($taskId)) { $paginator = new Util\Paginator(ITEMS_PER_PAGE, (int)($_GET['page'] ?? 1)); $paginator->setTotal($scheduler->taskRunTotal($taskId)); -$stats = $scheduler->taskRuntimeStats($taskId); echo $Twig->render('admin/scheduler/task.twig', [ - 'header' => $scheduler->heading(), - 'stats' => $stats, - 'duration' => json_encode($stats[0]['data']), - 'processed' => json_encode($stats[1]['data']), - 'task' => $scheduler->taskHistory( + 'header' => $scheduler->heading(), + 'stats' => $scheduler->taskRuntimeStats($taskId), + 'task' => $scheduler->taskHistory( $taskId, $paginator->limit(), $paginator->offset() ), 'paginator' => $paginator, diff --git a/sections/tools/development/periodic_stats.php b/sections/tools/development/periodic_stats.php index 6bc79d7fa..35a694e54 100644 --- a/sections/tools/development/periodic_stats.php +++ b/sections/tools/development/periodic_stats.php @@ -10,20 +10,7 @@ if (!$Viewer->permitted('admin_periodic_task_view')) { Error403::error(); } -$stats = new TaskScheduler()->runtimeStats(); echo $Twig->render('admin/scheduler/stats.twig', [ - 'hourly' => [ - 'duration' => json_encode($stats['hourly'][0]['data']), - 'processed' => json_encode($stats['hourly'][1]['data']), - ], - 'daily' => [ - 'duration' => json_encode($stats['daily'][0]['data']), - 'processed' => json_encode($stats['daily'][1]['data']), - ], - 'tasks' => [ - 'duration' => json_encode($stats['tasks'][0]['data']), - 'processed' => json_encode($stats['tasks'][1]['data']), - ], - 'totals' => $stats['totals'], + 'stats' => new TaskScheduler()->runtimeStats(), 'viewer' => $Viewer, ]); diff --git a/sections/tools/development/periodic_view.php b/sections/tools/development/periodic_view.php index 289a5ae2a..de38ab25e 100644 --- a/sections/tools/development/periodic_view.php +++ b/sections/tools/development/periodic_view.php @@ -18,10 +18,10 @@ if ($task) { Error403::error(); } authorize(); - $scheduler->runNow($task['periodic_task_id']); + $scheduler->runNow($task['id_task']); } elseif ($_REQUEST['mode'] === 'enqueue') { - $scheduler->enqueue($task['periodic_task_id']); - header("Location: tools.php?action=periodic&mode=view"); + $scheduler->enqueue($task['id_task']); + header("Location: ?action=periodic&mode=view"); exit; } } diff --git a/templates/admin/scheduler/edit.twig b/templates/admin/scheduler/edit.twig index 41fe96b4f..073c21759 100644 --- a/templates/admin/scheduler/edit.twig +++ b/templates/admin/scheduler/edit.twig @@ -9,8 +9,8 @@ {% endif %} - + @@ -21,15 +21,15 @@ {% for t in task_list %} - + - - - - - + + + + +
Name Class NameName Description Interval Enabled
- + {{ t.class_name }} - + diff --git a/templates/admin/scheduler/run.twig b/templates/admin/scheduler/run.twig index 707dfc88d..13847fb70 100644 --- a/templates/admin/scheduler/run.twig +++ b/templates/admin/scheduler/run.twig @@ -11,7 +11,7 @@
-Items processed: {{ processed|number_format }} +Items processed: {{ item_total|number_format }}
diff --git a/templates/admin/scheduler/stats.twig b/templates/admin/scheduler/stats.twig index 402716cdc..3a64f10d3 100644 --- a/templates/admin/scheduler/stats.twig +++ b/templates/admin/scheduler/stats.twig @@ -13,11 +13,11 @@ 		Errors
{{ totals.runs|number_format }}{{ totals.duration|number_format }} ms{{ totals.processed|number_format }}{{ totals.events|number_format }}{{ totals.errors|number_format }}{{ stats.totals.run_total|number_format }}{{ stats.totals.duration|number_format }} ms{{ stats.totals.item_total|number_format }}{{ stats.totals.event_total|number_format }}{{ stats.totals.error_total|number_format }}

@@ -30,85 +30,104 @@ {{ footer() }} diff --git a/templates/admin/scheduler/task.twig b/templates/admin/scheduler/task.twig index e34ab91d8..ca4d63649 100644 --- a/templates/admin/scheduler/task.twig +++ b/templates/admin/scheduler/task.twig @@ -5,7 +5,7 @@ {% include 'admin/scheduler/links.twig' with {'can_edit': viewer.permitted('admin_periodic_task_manage')} only %} {{ paginator.linkbox|raw }} {% for item in task.items %} - {% if loop.first %} +{% if loop.first %}
@@ -13,25 +13,25 @@ - + - - + + - {% endif %} +{% endif %} - - + + - {% for event in item.events %} - {% if loop.first %} +{% for event in item.events %} +{% if loop.first %} - {% endif %} - {% endfor %} - {% if loop.last %} +{% endif %} +{% endfor %} +{% if loop.last %}
{{ header|column('launchtime') }} Toggle{{ header|column('created') }} Toggle {{ header|column('duration') }} {{ header|column('status') }}{{ header|column('items') }}{{ header|column('errors') }}{{ header|column('item_total') }}{{ header|column('error_total') }}
- {{ item.launchTime|time_diff }} - + {{ item.created|time_diff }} + {{ item.duration }}ms {{ item.status }}{{ item.numItems|number_format }}{{ item.numErrors|number_format }}{{ item.itemTotal|number_format }}{{ item.errorTotal|number_format }}
@@ -41,39 +41,34 @@ - {% endif %} +{% endif %} - {% if loop.last %} +{% if loop.last %}
Event Reference
- {{ event.timestamp|time_diff }} - + {{ event.created|time_diff }} + {{ event.severity }} {{ event.event }} {{ event.reference }}
{{ paginator.linkbox|raw }} - {% endif %} +{% endif %} {% else %}

No history found

diff --git a/templates/admin/scheduler/view.twig b/templates/admin/scheduler/view.twig index 35e79ed00..72a183abf 100644 --- a/templates/admin/scheduler/view.twig +++ b/templates/admin/scheduler/view.twig @@ -45,12 +45,12 @@ date display between relative and absolute. {{ prefix }}{{ t.name }} + t.id_task }}">{{ prefix }}{{ t.name }} {{ t.period|time_compact }} {{ t.last_run|time_diff }} - + {{ t.duration }}ms @@ -58,20 +58,20 @@ date display between relative and absolute. Never {% else %} {{ t.next_run|time_diff }} - + {% endif %} {{ t.status|default('-') }} - {{ t.runs|number_format }} - {{ t.processed|number_format }} - {{ t.errors|number_format }} - {{ t.events|number_format }} + {{ t.run_total|number_format }} + {{ t.item_total|number_format }} + {{ t.error_total|number_format }} + {{ t.event_total|number_format }} {% if viewer.permitted('admin_schedule') %} Enqueue + viewer.auth }}&id={{ t.id_task }}&mode=enqueue">Enqueue Run + viewer.auth }}&id={{ t.id_task }}&mode=run">Run {% endif %} diff --git a/tests/phpunit/SchedulerTest.php b/tests/phpunit/SchedulerTest.php index b90a3a2ea..c2393ec1b 100644 --- a/tests/phpunit/SchedulerTest.php +++ b/tests/phpunit/SchedulerTest.php @@ -20,24 +20,24 @@ class SchedulerTest extends TestCase { public function testRunWithMissingImplementation(): void { $scheduler = new TaskScheduler(); $name = "RunUnimplemented"; - $db = DB::DB(); - $db->prepared_query(" - DELETE FROM periodic_task WHERE classname = ? - ", $name + $pg = new DB\Pg(PG_RW_DSN); + $pg->query(' + delete from task where class_name = $1 + ', $name ); - $db->prepared_query(" - INSERT INTO periodic_task - (classname, name, description, period) - VALUES (?, ?, ?, 86400) - ", + $pg->query(' + insert into task + (class_name, name, description, period) + VALUES ($1, $2, $3, 86400) + ', $name, "phpunit run task", "A run with no PHP implementation" ); - $this->expectOutputRegex('/^(?:\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \[(?:debug|info)\] (.*?)\n|Running task (?:.*?)\.\.\.DONE! \(\d+\.\d+\)\n)*$/'); + $this->expectOutputRegex('/^(?:\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \[(?:debug|info)\] (.*?)\n|Running task (?:.*?)\.\.\.Failed\.\n$)/'); $scheduler->run(); - $db->prepared_query(" - DELETE FROM periodic_task WHERE classname = ? - ", $name + $pg->query(' + delete from task where class_name = $1 + ', $name ); } @@ -62,15 +62,15 @@ class SchedulerTest extends TestCase { public function testMissingImplementation(): void { $scheduler = new TaskScheduler(); $name = "Unimplemented"; - $db = DB::DB(); - $db->prepared_query(" - DELETE FROM periodic_task WHERE classname = ? + $pg = new DB\Pg(PG_RW_DSN); + $pg->query(" + delete from task where class_name = $1 ", $name ); - $db->prepared_query(" - INSERT INTO periodic_task - (classname, name, description, period) - VALUES (?, ?, ?, 86400) + $pg->query(" + insert into task + (class_name, name, description, period) + VALUES ($1, $2, $3, 86400) ", $name, "phpunit task", "A task with no PHP implementation" ); @@ -81,8 +81,8 @@ class SchedulerTest extends TestCase { "sched-task-unimplemented" ); ob_end_clean(); - $db->prepared_query(" - DELETE FROM periodic_task WHERE classname = ? + $pg->query(" + delete from task where class_name = $1 ", $name ); } @@ -151,9 +151,9 @@ class SchedulerTest extends TestCase { $detail = current($list); $this->assertEqualsCanonicalizing( [ - "periodic_task_id", "name", "description", "period", - "is_enabled", "is_sane", "run_now", "runs", "processed", - "errors", "events", "duration", "status", "last_run", + "id_task", "name", "description", "period", + "is_enabled", "is_sane", "run_now", "run_total", "item_total", + "error_total", "event_total", "duration", "status", "last_run", "next_run", ], array_keys($detail), @@ -164,7 +164,7 @@ class SchedulerTest extends TestCase { public function testTaskHistory(): void { $scheduler = new TaskScheduler(); $task = $scheduler->findByName('Test'); - $taskId = $task['periodic_task_id']; + $taskId = $task['id_task']; $initial = $scheduler->taskHistory($taskId, 10, 0); $total = $scheduler->taskRunTotal($taskId); ob_start(); @@ -175,6 +175,7 @@ class SchedulerTest extends TestCase { $scheduler->taskRunTotal($taskId), 'task-run-total' ); + $this->assertTrue($scheduler->isLastRunIntervalOk(10), 'task-run-interval-ok'); $history = $scheduler->taskHistory($taskId, 10, 0); $this->assertEquals( @@ -183,7 +184,7 @@ class SchedulerTest extends TestCase { 'task-history-total' ); $item = current($history->items); - $this->assertTrue(Helper::recentDate($item->launchTime), 'task-launch-time'); + $this->assertTrue(Helper::recentDate($item->created->format('Y-m-d H:i:s')), 'task-created'); $this->assertEquals('completed', $item->status, 'task-status'); $this->assertEquals(0, $item->nrErrors, 'task-nr-error'); $this->assertEquals(0, $item->nrItems, 'task-nr-item'); @@ -192,7 +193,7 @@ class SchedulerTest extends TestCase { public function testTaskEnqueue(): void { $scheduler = new TaskScheduler(); $task = $scheduler->findByName('Test'); - $taskId = $task['periodic_task_id']; + $taskId = $task['id_task']; $this->assertEquals( 1, $scheduler->enqueue($taskId), @@ -215,14 +216,11 @@ class SchedulerTest extends TestCase { public function testTaskStats(): void { $scheduler = new TaskScheduler(); $task = $scheduler->findByName('Test'); - $taskId = $task['periodic_task_id']; - $stats = $scheduler->taskRuntimeStats($taskId, 1); - $this->assertCount(2, $stats, 'task-runtime-stats-count'); - $this->assertEquals( - 'duration', $stats[0]['name'], 'task-stats-duration', - ); - $this->assertEquals( - 'processed', $stats[1]['name'], 'task-stats-processed', + $taskId = $task['id_task']; + $this->assertEqualsCanonicalizing( + ['category', 'duration', 'item_total'], + array_keys($scheduler->taskRuntimeStats($taskId, 1)), + 'task-stats-duration', ); } @@ -236,26 +234,11 @@ class SchedulerTest extends TestCase { } public function testTaskUpdate(): void { - $this->assertEquals( - 0, - new TaskScheduler()->updateTask( - taskId: 1, - name: 'Test', - class: 'Bonkers', - description: 'Scheduler functionality test ' . randomString(), - period: 60, - isEnabled: false, - isSane: true, - isDebug: false, - ), - 'task-update-bad-class', - ); $this->assertEquals( 1, new TaskScheduler()->updateTask( taskId: 1, name: 'Test', - class: 'Test', description: 'Scheduler functionality test ' . randomString(), period: 60, isEnabled: false, @@ -267,7 +250,6 @@ class SchedulerTest extends TestCase { new TaskScheduler()->updateTask( taskId: 1, name: 'Test', - class: 'Test', description: 'Scheduler functionality test', period: 60, isEnabled: false, diff --git a/tests/phpunit/SiteInfoTest.php b/tests/phpunit/SiteInfoTest.php index 9a9f2cca3..1f8dd4176 100644 --- a/tests/phpunit/SiteInfoTest.php +++ b/tests/phpunit/SiteInfoTest.php @@ -12,6 +12,7 @@ class SiteInfoTest extends TestCase { $this->assertGreaterThanOrEqual(0, strlen($info->phpinfo()), 'siteinfo-phpinfo'); $this->assertCount(2, $info->uptime(), 'siteinfo-uptime'); $this->assertGreaterThan(0, count($info->phinx()), 'siteinfo-phinx'); + $this->assertEquals(0, $info->phinxMigrationsPending(), 'siteinfo-pending-phinx'); $this->assertStringStartsWith( 'Composer version 2.', $info->composerVersion(),