feat(worker, web): job ui v2 (#1608)

* wip on reposv2 table

* remove old repos table

* wip

* add banner

* remove repository carousel

* add example questions to chat page

* remove repo indexing job table

* add clear filter button

* add first sync banner

* remove permission job tables

* rename connection workload

* remove connection sync notification dot

* workload job return type plumbing

* add concept of repositoryDiscoveryIssueContext

* connections table

* replace existing connections table & rework what 'warning' means

* improve first time syncing banner

* change status badge behaviour in repos table subtly

* connection sync issue banner

* remove connection job table

* connection progress banner

* add clear filter button

* migrate other hosts to using report function

* changelog

* fix tests

* feedback

* feedback

* feedback

* feedback

* feedback

* move repo cleanup into sepreate queue with shared lock

* added additional deduplication behaviour

* add retry all button to repository table

* improve connection sync repo removal behaviour

Author: brendan-kellam

CHANGELOG.md

@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 - Migrated connection syncing, repository indexing, permission syncing, and background pruning from in-process managers and pollers to BullMQ workloads with retries and per-resource execution locking. [#1427](https://github.com/sourcebot-dev/sourcebot/pull/1427)
+- Refactored the repository and connection list view to display useful debugging information to owners, such as failure reasons and logs. [#1608](https://github.com/sourcebot-dev/sourcebot/pull/1608)
 
 ### Fixed
 - Upgraded Next.js to 16.3.1 to bound memory retained by high-cardinality dynamic route cache entries. [#1594](https://github.com/sourcebot-dev/sourcebot/pull/1594)

CLAUDE.md

@@ -32,8 +32,9 @@ Use the workload system in `packages/backend` for background work. Define the qu
 
 ### Lifecycle state
 
-- In `onStarted`, upsert the workload-specific job row as `IN_PROGRESS`. If the parent resource tracks a `latest...JobId`, update that pointer in the same database transaction.
-- Completion and terminal-failure hooks must always update their own historical job row by job ID. Do not condition that update on the job still being latest. Every job row should record its actual outcome.
+- BullMQ is the default source of truth for workload lifecycle state. Only persist a separate database job row when the product explicitly requires durable history beyond the queue's retention policy.
+- If a parent resource tracks a `latest...JobId`, update that pointer when the job starts so consumers can resolve its state from BullMQ.
+- For workloads that persist historical job rows, completion and terminal-failure hooks must always update their own row by job ID. Do not condition that update on the job still being latest. Every persisted job row should record its actual outcome.
 - `onTerminalFailure` only runs after the job exhausts all retry attempts. Intermediate failures are retried without marking the lifecycle row as terminally failed.
 - If a completion or failure hook publishes state onto the parent resource, use a conditional `updateMany` keyed by both the resource ID and its `latest...JobId`. This prevents an older hook from overwriting state belonging to a newer job after the execution lock has been released.
 - Parent-resource state written inside `process` is already serialized by the execution lock. It does not need a latest-job conditional merely because the resource tracks the latest job ID.

packages/backend/src/api.ts

@@ -7,7 +7,7 @@ import {
 } from '@bull-board/metrics';
 import { Octokit } from '@octokit/rest';
 import * as Sentry from "@sentry/node";
-import { PrismaClient, RepoIndexingJobType } from '@sourcebot/db';
+import { PrismaClient } from '@sourcebot/db';
 import { createLogger, env, JOB_PRIORITIES } from '@sourcebot/shared';
 import express, { NextFunction, Request, Response } from 'express';
 import 'express-async-errors';
@@ -176,7 +176,6 @@ const scheduleAndTriggerRepoIndexing = async ({
         reindexIntervalMs,
         {
             repoId,
-            type: RepoIndexingJobType.INDEX,
         },
         { priority: JOB_PRIORITIES.SCHEDULED },
     );
@@ -185,7 +184,6 @@ const scheduleAndTriggerRepoIndexing = async ({
         "repo-index",
         {
             repoId,
-            type: RepoIndexingJobType.INDEX,
         },
         { priority: JOB_PRIORITIES.INTERACTIVE },
     );

packages/backend/src/attachmentPruneWorkload.ts

@@ -18,12 +18,6 @@ interface Props {
     storage?: StorageBackend;
 }
 
-interface AttachmentPruneResult {
-    pendingClaimed: number;
-    committedClaimed: number;
-    reclaimed: number;
-}
-
 /**
  * Reclaims orphaned attachment blobs using the `DELETING` tombstone protocol:
  * an orphan is first atomically flipped to `DELETING`, then its bytes are
@@ -47,7 +41,7 @@ export const createAttachmentPruneWorkload = ({
     db,
     ttlHours,
     storage = getStorageBackend(),
-}: Props): Workload<"attachment-prune", AttachmentPruneResult> => ({
+}: Props): Workload<"attachment-prune"> => ({
     queueSpec: ATTACHMENT_PRUNE_QUEUE,
     concurrency: 1,
     ...(ttlHours > 0

packages/backend/src/azuredevops.test.ts

@@ -0,0 +1,137 @@
+import type { AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/azuredevops.type';
+import { beforeEach, describe, expect, test, vi } from 'vitest';
+
+const mocks = vi.hoisted(() => ({
+    getProjects: vi.fn(),
+    getRepositories: vi.fn(),
+    getRepository: vi.fn(),
+}));
+
+vi.mock("@sentry/node", () => ({
+    captureException: vi.fn(),
+}));
+
+vi.mock("@sourcebot/shared", async (importOriginal) => ({
+    ...await importOriginal<typeof import("@sourcebot/shared")>(),
+    getTokenFromConfig: vi.fn(async () => "token"),
+}));
+
+vi.mock("azure-devops-node-api", () => ({
+    getPersonalAccessTokenHandler: vi.fn(() => ({})),
+    WebApi: class {
+        getCoreApi = vi.fn(async () => ({
+            getProjects: mocks.getProjects,
+        }));
+        getGitApi = vi.fn(async () => ({
+            getRepositories: mocks.getRepositories,
+            getRepository: mocks.getRepository,
+        }));
+    },
+}));
+
+vi.mock("./utils.js", () => ({
+    fetchWithRetry: (routine: () => Promise<unknown>) => routine(),
+    measure: async (routine: () => Promise<unknown>) => ({
+        durationMs: 1,
+        data: await routine(),
+    }),
+}));
+
+import { getAzureDevOpsReposFromConfig } from './azuredevops';
+import { collectRepositoryDiscoveryIssues } from './repositoryDiscoveryIssueContext.js';
+
+const config = (overrides: Partial<AzureDevOpsConnectionConfig>): AzureDevOpsConnectionConfig => ({
+    type: "azuredevops",
+    deploymentType: "cloud",
+    token: { env: "AZURE_DEVOPS_TOKEN" },
+    ...overrides,
+});
+
+beforeEach(() => {
+    vi.clearAllMocks();
+    const notFound = Object.assign(new Error("Not Found"), { statusCode: 404 });
+    mocks.getProjects.mockRejectedValue(notFound);
+    mocks.getRepositories.mockRejectedValue(notFound);
+    mocks.getRepository.mockRejectedValue(notFound);
+});
+
+describe("Azure DevOps repository discovery", () => {
+    test("reports inaccessible configured targets as partial successes", async () => {
+        const result = await collectRepositoryDiscoveryIssues(() =>
+            getAzureDevOpsReposFromConfig(config({
+                orgs: ["missing-org"],
+                projects: ["org/missing-project"],
+                repos: ["org/project/missing-repo"],
+            }))
+        );
+
+        expect(result).toEqual({
+            value: [],
+            issues: [
+                {
+                    code: "NOT_FOUND_OR_INACCESSIBLE",
+                    effect: "TARGET_SKIPPED",
+                    subject: {
+                        kind: "organization",
+                        value: "missing-org",
+                    },
+                    message: "Azure DevOps organization was not found or is inaccessible.",
+                },
+                {
+                    code: "NOT_FOUND_OR_INACCESSIBLE",
+                    effect: "TARGET_SKIPPED",
+                    subject: {
+                        kind: "project",
+                        value: "org/missing-project",
+                    },
+                    message: "Azure DevOps project was not found or is inaccessible.",
+                },
+                {
+                    code: "NOT_FOUND_OR_INACCESSIBLE",
+                    effect: "TARGET_SKIPPED",
+                    subject: {
+                        kind: "repository",
+                        value: "org/project/missing-repo",
+                    },
+                    message: "Azure DevOps repository was not found or is inaccessible.",
+                },
+            ],
+        });
+    });
+
+    test("reports incomplete project enumeration within an organization", async () => {
+        mocks.getProjects.mockResolvedValue([
+            { name: "missing-id" },
+            { id: "broken-project-id", name: "broken-project" },
+        ]);
+        mocks.getRepositories.mockRejectedValue(new Error("Service unavailable"));
+
+        const result = await collectRepositoryDiscoveryIssues(() =>
+            getAzureDevOpsReposFromConfig(config({ orgs: ["my-org"] }))
+        );
+
+        expect(result).toEqual({
+            value: [],
+            issues: [
+                {
+                    code: "INVALID_PROVIDER_RESPONSE",
+                    effect: "DISCOVERY_INCOMPLETE",
+                    subject: {
+                        kind: "project",
+                        value: "my-org/missing-id",
+                    },
+                    message: "Azure DevOps returned a project without an ID, so its repositories were skipped.",
+                },
+                {
+                    code: "ENUMERATION_FAILED",
+                    effect: "DISCOVERY_INCOMPLETE",
+                    subject: {
+                        kind: "project",
+                        value: "my-org/broken-project",
+                    },
+                    message: "Azure DevOps repository enumeration did not complete for this project.",
+                },
+            ],
+        });
+    });
+});