mirror of
https://github.com/fosrl/pangolin.git
synced 2026-09-22 02:28:26 +02:00
Merge branch 'main' into dev
This commit is contained in:
Owen
+2
-2
@@ -2,8 +2,8 @@
|
||||
|
||||
import yargs from "yargs";
|
||||
import { hideBin } from "yargs/helpers";
|
||||
import { setAdminCredentials } from "@cli/commands/setAdminCredentials";
|
||||
import { resetUserSecurityKeys } from "@cli/commands/resetUserSecurityKeys";
|
||||
import { setAdminCredentials } from "./commands/setAdminCredentials";
|
||||
import { resetUserSecurityKeys } from "./commands/resetUserSecurityKeys";
|
||||
import { clearExitNodes } from "./commands/clearExitNodes";
|
||||
import { rotateServerSecret } from "./commands/rotateServerSecret";
|
||||
import { clearLicenseKeys } from "./commands/clearLicenseKeys";
|
||||
|
||||
+3
-3
@@ -459,13 +459,13 @@
|
||||
"searchApiKeys": "API sleutels zoeken...",
|
||||
"apiKeysAdd": "API sleutel genereren",
|
||||
"apiKeysErrorDelete": "Fout bij verwijderen API sleutel",
|
||||
"apiKeysErrorDeleteMessage": "Fout bij verwijderen API sleutel",
|
||||
"apiKeysErrorDeleteMessage": "Fout bij verwijderen API- leutel",
|
||||
"apiKeysQuestionRemove": "Weet u zeker dat u de API sleutel van de organisatie wilt verwijderen?",
|
||||
"apiKeysMessageRemove": "Eenmaal verwijderd, kan de APIsleutel niet meer worden gebruikt.",
|
||||
"apiKeysDeleteConfirm": "Bevestig Verwijderen API sleutel",
|
||||
"apiKeysDeleteConfirm": "Bevestig verwijderen API sleutel",
|
||||
"apiKeysDelete": "API sleutel verwijderen",
|
||||
"apiKeysManage": "API sleutels beheren",
|
||||
"apiKeysDescription": "API sleutels worden gebruikt om toegang te verifiëren met de integratie API ",
|
||||
"apiKeysDescription": "API sleutels worden gebruikt om te verifiëren met de integratie-API",
|
||||
"orgsManage": "Organisaties Beheren",
|
||||
"orgsDescription": "Bekijk en beheer alle organisaties op dit systeem",
|
||||
"provisioningKeysTitle": "Vertrekkende sleutel",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { formatBackupTimestamp } from "./backupFileName";
|
||||
import { formatBackupFileName, formatBackupTimestamp } from "./backupFileName";
|
||||
import { assertEquals } from "@test/assert";
|
||||
|
||||
// Local-time constructors are used throughout, matching formatBackupTimestamp,
|
||||
@@ -29,7 +29,9 @@ function testMonthIsOneIndexed() {
|
||||
}
|
||||
|
||||
{
|
||||
const result = formatBackupTimestamp(new Date(2026, 11, 31, 23, 59, 59));
|
||||
const result = formatBackupTimestamp(
|
||||
new Date(2026, 11, 31, 23, 59, 59)
|
||||
);
|
||||
assertEquals(
|
||||
result,
|
||||
"2026-12-31_23-59-59",
|
||||
@@ -73,9 +75,7 @@ function testNamesSortChronologically() {
|
||||
new Date(2026, 11, 31, 23, 59, 59)
|
||||
];
|
||||
|
||||
const sorted = taken
|
||||
.map((date) => formatBackupTimestamp(date))
|
||||
.sort();
|
||||
const sorted = taken.map((date) => formatBackupTimestamp(date)).sort();
|
||||
|
||||
assertEquals(
|
||||
sorted.join(","),
|
||||
@@ -89,11 +89,48 @@ function testNamesSortChronologically() {
|
||||
);
|
||||
}
|
||||
|
||||
function testFormatBackupFileName() {
|
||||
console.log("Running backup file name formatting tests...");
|
||||
|
||||
const date = new Date(2026, 8, 12, 20, 35, 56);
|
||||
|
||||
// With semver version string without leading 'v'
|
||||
assertEquals(
|
||||
formatBackupFileName("1.22.0", date),
|
||||
"db_2026-09-12_20-35-56_v1.22.0.sqlite",
|
||||
"Filename must include timestamp and prefixed version tag"
|
||||
);
|
||||
|
||||
// With version string already containing 'v'
|
||||
assertEquals(
|
||||
formatBackupFileName("v1.22.0", date),
|
||||
"db_2026-09-12_20-35-56_v1.22.0.sqlite",
|
||||
"Filename must not duplicate 'v' prefix if already present"
|
||||
);
|
||||
|
||||
// Without version (fallback/default)
|
||||
assertEquals(
|
||||
formatBackupFileName(undefined, date),
|
||||
"db_2026-09-12_20-35-56.sqlite",
|
||||
"Filename without version must match default timestamped format"
|
||||
);
|
||||
|
||||
// Distinct versions within the exact same second do not collide
|
||||
const sameSecondFile1 = formatBackupFileName("1.21.0", date);
|
||||
const sameSecondFile2 = formatBackupFileName("1.22.0", date);
|
||||
if (sameSecondFile1 === sameSecondFile2) {
|
||||
throw new Error(
|
||||
"Backup file names for different versions in the same second must not collide"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Run all tests
|
||||
try {
|
||||
testMonthIsOneIndexed();
|
||||
testEveryFieldIsZeroPadded();
|
||||
testNamesSortChronologically();
|
||||
testFormatBackupFileName();
|
||||
console.log("All tests passed successfully!");
|
||||
} catch (error) {
|
||||
console.error("Test failed:", error);
|
||||
|
||||
@@ -26,3 +26,26 @@ export function formatBackupTimestamp(date: Date = new Date()): string {
|
||||
|
||||
return `${datePart}_${timePart}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the full database backup file name, including timestamp and optional version tag.
|
||||
*
|
||||
* When a migration version is provided, the filename includes `_v<version>`,
|
||||
* preventing collisions between multiple migrations running in the same second and making it easy
|
||||
* to identify the migration state contained in the backup.
|
||||
*
|
||||
* @param version Optional migration version being run.
|
||||
* @param date The moment the backup is being taken. Defaults to now.
|
||||
* @returns A filename of the form `db_YYYY-MM-DD_HH-MM-SS_v<version>.sqlite` or `db_YYYY-MM-DD_HH-MM-SS.sqlite`.
|
||||
*/
|
||||
export function formatBackupFileName(
|
||||
version?: string,
|
||||
date: Date = new Date()
|
||||
): string {
|
||||
const timestamp = formatBackupTimestamp(date);
|
||||
if (version) {
|
||||
const versionTag = version.startsWith("v") ? version : `v${version}`;
|
||||
return `db_${timestamp}_${versionTag}.sqlite`;
|
||||
}
|
||||
return `db_${timestamp}.sqlite`;
|
||||
}
|
||||
|
||||
@@ -20,6 +20,11 @@ function getSegmentRegex(patternPart: string): RegExp {
|
||||
// resolves `.` / `..` segments, so a request like `/public%2F..%2Fadmin/`
|
||||
// or `/public/../admin/` is matched as `/admin/`, not as a literal segment
|
||||
// or a wildcard-swallowed sequence under `/public/*`.
|
||||
//
|
||||
// Applied to both the request path and the rule pattern: the pattern
|
||||
// validator only accepts spaces / non-ASCII in percent-encoded form, so a
|
||||
// rule like `/my%20docs/*` must be compared against the decoded segment
|
||||
// `my docs`, not the literal text `my%20docs`.
|
||||
function decodeAndResolvePath(p: string): string[] {
|
||||
const rawParts = p.split("/").filter(Boolean);
|
||||
|
||||
@@ -48,7 +53,7 @@ function decodeAndResolvePath(p: string): string[] {
|
||||
}
|
||||
|
||||
export function isPathAllowed(pattern: string, path: string): boolean {
|
||||
const patternParts = pattern.split("/").filter(Boolean);
|
||||
const patternParts = decodeAndResolvePath(pattern);
|
||||
const pathParts = decodeAndResolvePath(path);
|
||||
|
||||
function matchSegments(
|
||||
|
||||
@@ -70,13 +70,8 @@ export async function verifyApiKeyAccess(
|
||||
);
|
||||
}
|
||||
|
||||
if (!apiKeyOrg.orgId) {
|
||||
return next(
|
||||
createHttpError(
|
||||
HttpCode.INTERNAL_SERVER_ERROR,
|
||||
`API key with ID ${apiKeyId} does not have an organization ID`
|
||||
)
|
||||
);
|
||||
if (!apiKey.apiKeyOrg?.orgId) {
|
||||
return next(createHttpError(HttpCode.INTERNAL_SERVER_ERROR, `API key with ID ${apiKeyId} does not have an organization ID`));
|
||||
}
|
||||
|
||||
if (!req.userOrg) {
|
||||
@@ -86,7 +81,7 @@ export async function verifyApiKeyAccess(
|
||||
.where(
|
||||
and(
|
||||
eq(userOrgs.userId, userId),
|
||||
eq(userOrgs.orgId, apiKeyOrg.orgId)
|
||||
eq(userOrgs.orgId, apiKey.apiKeyOrg.orgId)
|
||||
)
|
||||
)
|
||||
.limit(1);
|
||||
|
||||
@@ -97,7 +97,7 @@ export async function exportConnectionAuditLogs(
|
||||
|
||||
const baseQuery = queryConnection(data);
|
||||
|
||||
const log = await baseQuery.limit(data.limit).offset(data.offset);
|
||||
const log = await baseQuery.limit(MAX_EXPORT_LIMIT);
|
||||
|
||||
const csvData = generateCSV(log);
|
||||
|
||||
|
||||
@@ -386,6 +386,38 @@ function runSpecialCharacterTests() {
|
||||
console.log("All special character tests passed!");
|
||||
}
|
||||
|
||||
function runEncodedPatternTests() {
|
||||
console.log("\nRunning percent-encoded pattern tests...");
|
||||
|
||||
// isValidUrlGlobPattern accepts percent-encoded sequences and rejects
|
||||
// raw spaces / non-ASCII, so `%20` and `%C3%A9` are the only way to write
|
||||
// a PATH rule for such a path. Badger sends the request path already
|
||||
// decoded (Go's req.URL.Path), and isPathAllowed decodes it again, so the
|
||||
// rule pattern must be decoded the same way or it can never match.
|
||||
assertEquals(
|
||||
isPathAllowed("/my%20docs/*", "/my docs/report.pdf"),
|
||||
true,
|
||||
"Percent-encoded space in pattern should match decoded request path"
|
||||
);
|
||||
assertEquals(
|
||||
isPathAllowed("/my%20docs/*", "/my%20docs/report.pdf"),
|
||||
true,
|
||||
"Percent-encoded space in pattern should match raw-encoded request path"
|
||||
);
|
||||
assertEquals(
|
||||
isPathAllowed("/caf%C3%A9", "/café"),
|
||||
true,
|
||||
"Percent-encoded UTF-8 in pattern should match decoded request path"
|
||||
);
|
||||
assertEquals(
|
||||
isPathAllowed("/my%20docs/*", "/my-docs/report.pdf"),
|
||||
false,
|
||||
"Decoded pattern must still reject a different path"
|
||||
);
|
||||
|
||||
console.log("All percent-encoded pattern tests passed!");
|
||||
}
|
||||
|
||||
function runRegionTests() {
|
||||
console.log("\nRunning isIpInRegion tests...");
|
||||
|
||||
@@ -446,6 +478,7 @@ function runRegionTests() {
|
||||
try {
|
||||
runTests();
|
||||
runSpecialCharacterTests();
|
||||
runEncodedPatternTests();
|
||||
runRegionTests();
|
||||
console.log("\n✅ All tests passed!");
|
||||
} catch (error) {
|
||||
|
||||
@@ -31,7 +31,7 @@ export async function addPeer(
|
||||
.where(eq(newts.siteId, siteId))
|
||||
.limit(1);
|
||||
if (!newt) {
|
||||
throw new Error(`Site found for site ${siteId}`);
|
||||
throw new Error(`Newt not found for site ${siteId}`);
|
||||
}
|
||||
newtId = newt.newtId;
|
||||
}
|
||||
|
||||
@@ -509,7 +509,8 @@ async function updateHttpResource(
|
||||
}
|
||||
|
||||
// catch when the resource policy changes or gets cleared
|
||||
if (resource.resourcePolicyId != updateData.resourcePolicyId) {
|
||||
if (updateData.resourcePolicyId !== undefined &&
|
||||
resource.resourcePolicyId !== updateData.resourcePolicyId) {
|
||||
await clearResourceSpecificSettings(
|
||||
resource.resourceId,
|
||||
resource.orgId,
|
||||
|
||||
@@ -263,7 +263,7 @@ export async function createSite(
|
||||
const { value: newClientAddress, release } =
|
||||
await getNextAvailableClientSubnet(orgId);
|
||||
releaseSubnetLock = release;
|
||||
updatedAddress = newClientAddress.split("/")[0];
|
||||
updatedAddress = `${newClientAddress.split("/")[0]}/${org.subnet ? org.subnet.split("/")[1] : "32"}`;
|
||||
}
|
||||
|
||||
let newSite: Site | undefined;
|
||||
|
||||
@@ -113,7 +113,7 @@ export async function updateSite(
|
||||
.where(
|
||||
and(
|
||||
eq(sites.niceId, updateData.niceId),
|
||||
eq(sites.orgId, sites.orgId),
|
||||
eq(sites.orgId, existingSite.orgId),
|
||||
ne(sites.siteId, siteId)
|
||||
)
|
||||
)
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
import { assertEquals } from "@test/assert";
|
||||
import { getSiteResourceParamsSchema } from "./getSiteResource";
|
||||
|
||||
function testSiteResourceIdOnlyParams() {
|
||||
const result = getSiteResourceParamsSchema.safeParse({
|
||||
siteResourceId: "42"
|
||||
});
|
||||
|
||||
assertEquals(
|
||||
result.success,
|
||||
true,
|
||||
"siteResourceId-only integration routes should pass validation"
|
||||
);
|
||||
|
||||
if (result.success) {
|
||||
assertEquals(
|
||||
result.data.siteResourceId,
|
||||
42,
|
||||
"siteResourceId should be parsed as a number"
|
||||
);
|
||||
assertEquals(
|
||||
result.data.orgId,
|
||||
undefined,
|
||||
"orgId should remain optional"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function testOrgScopedParamsRemainSupported() {
|
||||
const result = getSiteResourceParamsSchema.safeParse({
|
||||
siteResourceId: "42",
|
||||
orgId: "org-id"
|
||||
});
|
||||
|
||||
assertEquals(
|
||||
result.success,
|
||||
true,
|
||||
"org-scoped routes should continue to pass validation"
|
||||
);
|
||||
}
|
||||
|
||||
function testInvalidSiteResourceId() {
|
||||
const result = getSiteResourceParamsSchema.safeParse({
|
||||
siteResourceId: "not-a-number"
|
||||
});
|
||||
|
||||
assertEquals(
|
||||
result.success,
|
||||
false,
|
||||
"non-numeric siteResourceIds should fail validation"
|
||||
);
|
||||
}
|
||||
|
||||
testSiteResourceIdOnlyParams();
|
||||
testOrgScopedParamsRemainSupported();
|
||||
testInvalidSiteResourceId();
|
||||
|
||||
console.log("All getSiteResource parameter validation tests passed.");
|
||||
@@ -10,7 +10,7 @@ import { fromError } from "zod-validation-error";
|
||||
import logger from "@server/logger";
|
||||
import { OpenAPITags, registry } from "@server/openApi";
|
||||
|
||||
const getSiteResourceParamsSchema = z.strictObject({
|
||||
export const getSiteResourceParamsSchema = z.strictObject({
|
||||
siteResourceId: z
|
||||
.string()
|
||||
.optional()
|
||||
@@ -22,15 +22,17 @@ const getSiteResourceParamsSchema = z.strictObject({
|
||||
});
|
||||
|
||||
async function query(siteResourceId?: number, niceId?: string, orgId?: string) {
|
||||
if (siteResourceId && orgId) {
|
||||
if (siteResourceId) {
|
||||
const [siteResource] = await db
|
||||
.select()
|
||||
.from(siteResources)
|
||||
.where(
|
||||
and(
|
||||
orgId
|
||||
? and(
|
||||
eq(siteResources.siteResourceId, siteResourceId),
|
||||
eq(siteResources.orgId, orgId)
|
||||
)
|
||||
: eq(siteResources.siteResourceId, siteResourceId)
|
||||
)
|
||||
.limit(1);
|
||||
return siteResource;
|
||||
|
||||
@@ -0,0 +1,342 @@
|
||||
import { execFileSync } from "child_process";
|
||||
import fs from "fs";
|
||||
import os from "os";
|
||||
import path from "path";
|
||||
import { fileURLToPath } from "url";
|
||||
import Database from "better-sqlite3";
|
||||
import { assertEquals } from "@test/assert";
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const repoRoot = path.resolve(here, "..", "..");
|
||||
const migrationsScript = path.join(here, "migrationsSqlite.ts");
|
||||
|
||||
const SEED_STATEMENTS = [
|
||||
`CREATE TABLE versionMigrations (version TEXT PRIMARY KEY, executedAt INTEGER NOT NULL)`,
|
||||
`INSERT INTO versionMigrations (version, executedAt) VALUES ('1.21.0', 1750000000000)`,
|
||||
`CREATE TABLE sites (siteId INTEGER PRIMARY KEY AUTOINCREMENT, subnet TEXT)`,
|
||||
`INSERT INTO sites (subnet) VALUES ('10.0.0.0/24')`,
|
||||
`CREATE TABLE roles (roleId INTEGER PRIMARY KEY AUTOINCREMENT, orgId TEXT, isAdmin INTEGER DEFAULT 0, sshSudoMode TEXT DEFAULT 'none')`,
|
||||
`INSERT INTO roles (orgId, isAdmin, sshSudoMode) VALUES ('org1', 0, 'none')`,
|
||||
`CREATE TABLE licenseKey (licenseKeyId INTEGER PRIMARY KEY AUTOINCREMENT)`,
|
||||
`CREATE TABLE targets (targetId INTEGER PRIMARY KEY AUTOINCREMENT, resourceId INTEGER, siteId INTEGER NOT NULL, ip TEXT NOT NULL, method TEXT, port INTEGER NOT NULL, internalPort INTEGER, enabled INTEGER DEFAULT 1, path TEXT, pathMatchType TEXT, rewritePath TEXT, rewritePathType TEXT, priority INTEGER DEFAULT 100, mode TEXT DEFAULT 'http', authToken TEXT)`,
|
||||
`CREATE TABLE subscriptions (subscriptionId INTEGER PRIMARY KEY AUTOINCREMENT)`,
|
||||
`CREATE TABLE clients (clientId INTEGER PRIMARY KEY AUTOINCREMENT)`,
|
||||
`CREATE TABLE orgs (orgId TEXT PRIMARY KEY)`,
|
||||
`INSERT INTO orgs (orgId) VALUES ('org1')`,
|
||||
`CREATE TABLE siteResources (siteResourceId INTEGER PRIMARY KEY AUTOINCREMENT)`,
|
||||
`CREATE TABLE eventStreamingDestinations (destinationId INTEGER PRIMARY KEY AUTOINCREMENT)`,
|
||||
`CREATE TABLE roleActions (roleId INTEGER, actionId TEXT, orgId TEXT)`,
|
||||
`CREATE TABLE newt (newtId INTEGER PRIMARY KEY AUTOINCREMENT)`
|
||||
];
|
||||
|
||||
function seedDatabase(dbPath: string) {
|
||||
const db = new Database(dbPath);
|
||||
try {
|
||||
for (const statement of SEED_STATEMENTS) {
|
||||
db.exec(statement);
|
||||
}
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
function tableColumns(dbPath: string, tableName: string): string[] {
|
||||
const db = new Database(dbPath, { readonly: true });
|
||||
try {
|
||||
return (
|
||||
db.prepare(`PRAGMA table_info(${tableName})`).all() as Array<{
|
||||
name: unknown;
|
||||
}>
|
||||
).map((row) => String(row.name));
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
function executedMigrationVersions(dbPath: string): string[] {
|
||||
const db = new Database(dbPath, { readonly: true });
|
||||
try {
|
||||
return (
|
||||
db.prepare(`SELECT version FROM versionMigrations`).all() as Array<{
|
||||
version: unknown;
|
||||
}>
|
||||
).map((row) => String(row.version));
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
function runMigrations(
|
||||
workdir: string,
|
||||
env: Record<string, string> = {}
|
||||
): {
|
||||
exitCode: number;
|
||||
output: string;
|
||||
} {
|
||||
const tsconfig = ["tsconfig.json", "tsconfig.oss.json"]
|
||||
.map((file) => path.join(repoRoot, file))
|
||||
.find((file) => fs.existsSync(file));
|
||||
if (!tsconfig) {
|
||||
throw new Error("No tsconfig found for @server path aliases");
|
||||
}
|
||||
const tsxCli = path.join(
|
||||
repoRoot,
|
||||
"node_modules",
|
||||
"tsx",
|
||||
"dist",
|
||||
"cli.mjs"
|
||||
);
|
||||
if (!fs.existsSync(tsxCli)) {
|
||||
throw new Error("tsx is not installed; run npm ci first");
|
||||
}
|
||||
try {
|
||||
const output = execFileSync(
|
||||
process.execPath,
|
||||
[tsxCli, "--tsconfig", tsconfig, migrationsScript],
|
||||
{
|
||||
cwd: workdir,
|
||||
timeout: 120000,
|
||||
encoding: "utf8",
|
||||
env: { ...process.env, NODE_ENV: "test", ...env }
|
||||
}
|
||||
);
|
||||
return { exitCode: 0, output };
|
||||
} catch (error) {
|
||||
const output =
|
||||
error instanceof Error
|
||||
? (error as Error & { stdout?: unknown }).stdout
|
||||
: "";
|
||||
return { exitCode: 1, output: String(output ?? "") };
|
||||
}
|
||||
}
|
||||
|
||||
function createTestEnvironment(): string {
|
||||
for (const generated of ["server/build.ts", "server/db/index.ts"]) {
|
||||
if (!fs.existsSync(path.join(repoRoot, generated))) {
|
||||
throw new Error(
|
||||
`Missing ${generated}; run npm run set:oss && npm run set:sqlite first`
|
||||
);
|
||||
}
|
||||
}
|
||||
const workdir = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), "pangolin-backup-test-")
|
||||
);
|
||||
fs.mkdirSync(path.join(workdir, "config", "db"), { recursive: true });
|
||||
fs.copyFileSync(
|
||||
path.join(repoRoot, "config", "config.example.yml"),
|
||||
path.join(workdir, "config", "config.yml")
|
||||
);
|
||||
const traefikSrc = path.join(repoRoot, "config", "traefik");
|
||||
if (fs.existsSync(traefikSrc)) {
|
||||
fs.cpSync(traefikSrc, path.join(workdir, "config", "traefik"), {
|
||||
recursive: true
|
||||
});
|
||||
}
|
||||
fs.symlinkSync(
|
||||
path.join(repoRoot, "server"),
|
||||
path.join(workdir, "server"),
|
||||
process.platform === "win32" ? "junction" : "dir"
|
||||
);
|
||||
return workdir;
|
||||
}
|
||||
|
||||
function testMultipleSequentialMigrations() {
|
||||
console.log("Running multiple sequential migrations test...");
|
||||
const workdir = createTestEnvironment();
|
||||
try {
|
||||
seedDatabase(path.join(workdir, "config", "db", "db.sqlite"));
|
||||
const result = runMigrations(workdir);
|
||||
assertEquals(result.exitCode, 0, "Seeded migrations must run cleanly");
|
||||
if (!result.output.includes("All migrations completed successfully")) {
|
||||
throw new Error(
|
||||
"Seeded migrations did not complete; the backup assertions below would be vacuous"
|
||||
);
|
||||
}
|
||||
|
||||
const backupsDir = path.join(workdir, "config", "db", "backups");
|
||||
const backups = fs.existsSync(backupsDir)
|
||||
? fs
|
||||
.readdirSync(backupsDir)
|
||||
.filter((file) => file.endsWith(".sqlite"))
|
||||
: [];
|
||||
|
||||
// Upgrading from 1.21.0 runs 1.22.0 and 1.23.0 -> produces 2 distinct backups
|
||||
assertEquals(
|
||||
backups.length,
|
||||
2,
|
||||
"Each migration must have its own distinct backup snapshot"
|
||||
);
|
||||
|
||||
const v122Backup = backups.find((file) =>
|
||||
file.includes("_v1.22.0.sqlite")
|
||||
);
|
||||
const v123Backup = backups.find((file) =>
|
||||
file.includes("_v1.23.0.sqlite")
|
||||
);
|
||||
|
||||
if (!v122Backup || !v123Backup) {
|
||||
throw new Error(
|
||||
`Expected backups for v1.22.0 and v1.23.0, found: ${backups.join(", ")}`
|
||||
);
|
||||
}
|
||||
|
||||
// Verify pre-1.22.0 snapshot state: sites has 'subnet' (not exitNodeSubnet), versions = [1.21.0]
|
||||
const v122Columns = tableColumns(
|
||||
path.join(backupsDir, v122Backup),
|
||||
"sites"
|
||||
);
|
||||
assertEquals(
|
||||
v122Columns.includes("subnet") &&
|
||||
!v122Columns.includes("exitNodeSubnet"),
|
||||
true,
|
||||
"Backup before 1.22.0 must retain pre-1.22.0 schema (sites.subnet)"
|
||||
);
|
||||
const v122Versions = executedMigrationVersions(
|
||||
path.join(backupsDir, v122Backup)
|
||||
);
|
||||
assertEquals(
|
||||
v122Versions.includes("1.21.0") && !v122Versions.includes("1.22.0"),
|
||||
true,
|
||||
"Backup before 1.22.0 must only record version 1.21.0"
|
||||
);
|
||||
|
||||
// Verify pre-1.23.0 snapshot state: sites has 'exitNodeSubnet' (1.22.0 applied), newt has no agent
|
||||
const v123Columns = tableColumns(
|
||||
path.join(backupsDir, v123Backup),
|
||||
"sites"
|
||||
);
|
||||
assertEquals(
|
||||
v123Columns.includes("exitNodeSubnet"),
|
||||
true,
|
||||
"Backup before 1.23.0 must contain successfully applied 1.22.0 schema (sites.exitNodeSubnet)"
|
||||
);
|
||||
const v123NewtCols = tableColumns(
|
||||
path.join(backupsDir, v123Backup),
|
||||
"newt"
|
||||
);
|
||||
assertEquals(
|
||||
!v123NewtCols.includes("agent"),
|
||||
true,
|
||||
"Backup before 1.23.0 must not contain 1.23.0 schema changes yet"
|
||||
);
|
||||
const v123Versions = executedMigrationVersions(
|
||||
path.join(backupsDir, v123Backup)
|
||||
);
|
||||
assertEquals(
|
||||
v123Versions.includes("1.21.0") && v123Versions.includes("1.22.0"),
|
||||
true,
|
||||
"Backup before 1.23.0 must record both 1.21.0 and 1.22.0"
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(workdir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function testFailureInLaterMigrationPreservesRestorePoints() {
|
||||
console.log("Running failure in later migration test...");
|
||||
const workdir = createTestEnvironment();
|
||||
try {
|
||||
const dbPath = path.join(workdir, "config", "db", "db.sqlite");
|
||||
seedDatabase(dbPath);
|
||||
|
||||
// Intentionally drop table 'newt' so migration 1.23.0 fails on ALTER TABLE newt ADD COLUMN agent
|
||||
const db = new Database(dbPath);
|
||||
db.exec("DROP TABLE newt;");
|
||||
db.close();
|
||||
|
||||
const result = runMigrations(workdir);
|
||||
assertEquals(
|
||||
result.exitCode,
|
||||
1,
|
||||
"Migration suite must fail when 1.23.0 errors"
|
||||
);
|
||||
|
||||
const backupsDir = path.join(workdir, "config", "db", "backups");
|
||||
const backups = fs.existsSync(backupsDir)
|
||||
? fs
|
||||
.readdirSync(backupsDir)
|
||||
.filter((file) => file.endsWith(".sqlite"))
|
||||
: [];
|
||||
|
||||
// Both pre-1.22.0 and pre-1.23.0 backups must exist
|
||||
assertEquals(
|
||||
backups.length,
|
||||
2,
|
||||
"Backups for earlier successful migration and the failed migration must both exist"
|
||||
);
|
||||
|
||||
const v122Backup = backups.find((file) =>
|
||||
file.includes("_v1.22.0.sqlite")
|
||||
);
|
||||
const v123Backup = backups.find((file) =>
|
||||
file.includes("_v1.23.0.sqlite")
|
||||
);
|
||||
|
||||
if (!v122Backup || !v123Backup) {
|
||||
throw new Error(
|
||||
`Expected restore points for v1.22.0 and v1.23.0, found: ${backups.join(", ")}`
|
||||
);
|
||||
}
|
||||
|
||||
// Verify pre-1.23.0 backup is a valid restore point with 1.22.0 changes applied
|
||||
const v123SitesCols = tableColumns(
|
||||
path.join(backupsDir, v123Backup),
|
||||
"sites"
|
||||
);
|
||||
assertEquals(
|
||||
v123SitesCols.includes("exitNodeSubnet"),
|
||||
true,
|
||||
"Pre-failure restore point must have 1.22.0 changes intact"
|
||||
);
|
||||
const v123Versions = executedMigrationVersions(
|
||||
path.join(backupsDir, v123Backup)
|
||||
);
|
||||
assertEquals(
|
||||
v123Versions.includes("1.22.0"),
|
||||
true,
|
||||
"Pre-failure restore point must record successful 1.22.0 migration"
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(workdir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function testDisableBackupOnMigration() {
|
||||
console.log("Running DISABLE_BACKUP_ON_MIGRATION test...");
|
||||
const workdir = createTestEnvironment();
|
||||
try {
|
||||
seedDatabase(path.join(workdir, "config", "db", "db.sqlite"));
|
||||
const result = runMigrations(workdir, {
|
||||
DISABLE_BACKUP_ON_MIGRATION: "1"
|
||||
});
|
||||
assertEquals(
|
||||
result.exitCode,
|
||||
0,
|
||||
"Migrations must succeed with backups disabled"
|
||||
);
|
||||
|
||||
const backupsDir = path.join(workdir, "config", "db", "backups");
|
||||
const backups = fs.existsSync(backupsDir)
|
||||
? fs
|
||||
.readdirSync(backupsDir)
|
||||
.filter((file) => file.endsWith(".sqlite"))
|
||||
: [];
|
||||
assertEquals(
|
||||
backups.length,
|
||||
0,
|
||||
"No backup files should be created when DISABLE_BACKUP_ON_MIGRATION is set"
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(workdir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
testMultipleSequentialMigrations();
|
||||
testFailureInLaterMigrationPreservesRestorePoints();
|
||||
testDisableBackupOnMigration();
|
||||
console.log("All backup migration regression tests passed successfully!");
|
||||
} catch (error) {
|
||||
console.error("Test failed:", error);
|
||||
process.exit(1);
|
||||
}
|
||||
@@ -5,7 +5,7 @@ import path from "path";
|
||||
import semver from "semver";
|
||||
import { versionMigrations } from "../db/sqlite";
|
||||
import { __DIRNAME, APP_PATH, APP_VERSION } from "@server/lib/consts";
|
||||
import { formatBackupTimestamp } from "@server/lib/backupFileName";
|
||||
import { formatBackupFileName } from "@server/lib/backupFileName";
|
||||
import { SqliteError } from "better-sqlite3";
|
||||
import fs from "fs";
|
||||
import { build } from "@server/build";
|
||||
@@ -107,7 +107,7 @@ async function run() {
|
||||
await runMigrations();
|
||||
}
|
||||
|
||||
function backupDb() {
|
||||
function backupDb(version?: string) {
|
||||
// make dir config/db/backups
|
||||
const appPath = APP_PATH;
|
||||
const dbDir = path.join(appPath, "db");
|
||||
@@ -120,11 +120,10 @@ function backupDb() {
|
||||
}
|
||||
|
||||
// copy the db.sqlite file to backups
|
||||
// add the date to the filename
|
||||
const date = new Date();
|
||||
const dateString = formatBackupTimestamp(date);
|
||||
// add the date and migration version to the filename
|
||||
const fileName = formatBackupFileName(version);
|
||||
const dbPath = path.join(dbDir, "db.sqlite");
|
||||
const backupPath = path.join(backupsDir, `db_${dateString}.sqlite`);
|
||||
const backupPath = path.join(backupsDir, fileName);
|
||||
fs.copyFileSync(dbPath, backupPath);
|
||||
}
|
||||
|
||||
@@ -163,6 +162,12 @@ export async function runMigrations() {
|
||||
}
|
||||
} catch (e) {
|
||||
console.error("Error running migrations:", e);
|
||||
if (
|
||||
process.env.NODE_ENV === "test" ||
|
||||
process.env.ENVIRONMENT === "test"
|
||||
) {
|
||||
throw e;
|
||||
}
|
||||
await new Promise((resolve) =>
|
||||
setTimeout(resolve, 1000 * 60 * 60 * 24 * 1)
|
||||
);
|
||||
@@ -197,7 +202,7 @@ async function executeScripts() {
|
||||
try {
|
||||
if (!process.env.DISABLE_BACKUP_ON_MIGRATION) {
|
||||
// Backup the database before running the migration
|
||||
backupDb();
|
||||
backupDb(migration.version);
|
||||
}
|
||||
|
||||
await migration.run();
|
||||
|
||||
@@ -10,7 +10,6 @@ import {
|
||||
users
|
||||
} from "../../db/sqlite";
|
||||
import { APP_PATH, configFilePath1, configFilePath2 } from "@server/lib/consts";
|
||||
import { formatBackupTimestamp } from "@server/lib/backupFileName";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import fs from "fs";
|
||||
import * as yaml from "js-yaml";
|
||||
@@ -21,25 +20,6 @@ import { fromZodError } from "zod-validation-error";
|
||||
export default async function migration() {
|
||||
console.log("Running setup script 1.0.0-beta.9...");
|
||||
|
||||
// make dir config/db/backups
|
||||
const appPath = APP_PATH;
|
||||
const dbDir = path.join(appPath, "db");
|
||||
|
||||
const backupsDir = path.join(dbDir, "backups");
|
||||
|
||||
// check if the backups directory exists and create it if it doesn't
|
||||
if (!fs.existsSync(backupsDir)) {
|
||||
fs.mkdirSync(backupsDir, { recursive: true });
|
||||
}
|
||||
|
||||
// copy the db.sqlite file to backups
|
||||
// add the date to the filename
|
||||
const date = new Date();
|
||||
const dateString = formatBackupTimestamp(date);
|
||||
const dbPath = path.join(dbDir, "db.sqlite");
|
||||
const backupPath = path.join(backupsDir, `db_${dateString}.sqlite`);
|
||||
fs.copyFileSync(dbPath, backupPath);
|
||||
|
||||
await db.transaction(async (trx) => {
|
||||
try {
|
||||
// Determine which config file exists
|
||||
|
||||
Reference in New Issue
Block a user