calvin.erfmann/pangolin
1
0
Fork 0
mirror of https://github.com/fosrl/pangolin.git synced 2026-01-28 22:00:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make easier to run in dev - fix a couple of things

Browse Source
This commit is contained in:
Owen
2025-10-12 16:23:38 -07:00
parent f17a957058
commit a50c0d84e9
10 changed files with 265 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
drizzle.pg.config.ts
View File

@@ -4,7 +4,7 @@ import { build } from "@server/build";
const schema = [
path.join("server", "db", "pg", "schema.ts"),
path.join("server", "db", "pg", "pSchema.ts")
path.join("server", "db", "pg", "privateSchema.ts")
];
export default defineConfig({

2
drizzle.sqlite.config.ts
View File

@@ -5,7 +5,7 @@ import path from "path";
const schema = [
path.join("server", "db", "sqlite", "schema.ts"),
path.join("server", "db", "sqlite", "pSchema.ts")
path.join("server", "db", "sqlite", "privateSchema.ts")
];
export default defineConfig({

6
messages/en-US.json
View File

@@ -1542,8 +1542,8 @@
"autoLoginError": "Auto Login Error",
"autoLoginErrorNoRedirectUrl": "No redirect URL received from the identity provider.",
"autoLoginErrorGeneratingUrl": "Failed to generate authentication URL.",
"remoteExitNodeManageRemoteExitNodes": "Managed Nodes",
"remoteExitNodeDescription": "Self-host one or more nodes for tunnel exit servers",
"remoteExitNodeManageRemoteExitNodes": "Remote Nodes",
"remoteExitNodeDescription": "Self-host one or more remote nodes for tunnel exit servers",
"remoteExitNodes": "Nodes",
"searchRemoteExitNodes": "Search nodes...",
"remoteExitNodeAdd": "Add Node",
@@ -1553,7 +1553,7 @@
"remoteExitNodeMessageConfirm": "To confirm, please type the name of the node below.",
"remoteExitNodeConfirmDelete": "Confirm Delete Node",
"remoteExitNodeDelete": "Delete Node",
"sidebarRemoteExitNodes": "Nodes",
"sidebarRemoteExitNodes": "Remote Nodes",
"remoteExitNodeCreate": {
"title": "Create Node",
"description": "Create a new node to extend your network connectivity",

34
server/apiServer.ts
View File

@@ -44,27 +44,25 @@ export function createApiServer() {
}
const corsConfig = config.getRawConfig().server.cors;
const options = {
...(corsConfig?.origins
? { origin: corsConfig.origins }
: {
origin: (origin: any, callback: any) => {
callback(null, true);
}
}),
...(corsConfig?.methods && { methods: corsConfig.methods }),
...(corsConfig?.allowed_headers && {
allowedHeaders: corsConfig.allowed_headers
}),
credentials: !(corsConfig?.credentials === false)
};
if (build == "oss") {
const options = {
...(corsConfig?.origins
? { origin: corsConfig.origins }
: {
origin: (origin: any, callback: any) => {
callback(null, true);
}
}),
...(corsConfig?.methods && { methods: corsConfig.methods }),
...(corsConfig?.allowed_headers && {
allowedHeaders: corsConfig.allowed_headers
}),
credentials: !(corsConfig?.credentials === false)
};
if (build == "oss" || !corsConfig) {
logger.debug("Using CORS options", options);
apiServer.use(cors(options));
} else {
} else if (corsConfig) {
// Use the custom CORS middleware with loginPage support
apiServer.use(corsWithLoginPageSupport(corsConfig));
}

206
server/lib/billing/usageService.ts
View File

@@ -31,6 +31,17 @@ interface StripeEvent {
};
}
export function noop() {
if (
build !== "saas" ||
!process.env.S3_BUCKET ||
!process.env.LOCAL_FILE_PATH
) {
return true;
}
return false;
}
export class UsageService {
private cache: NodeCache;
private bucketName: string | undefined;
@@ -41,7 +52,7 @@ export class UsageService {
constructor() {
this.cache = new NodeCache({ stdTTL: 300 }); // 5 minute TTL
if (build !== "saas") {
if (noop()) {
return;
}
// this.bucketName = privateConfig.getRawPrivateConfig().stripe?.s3Bucket;
@@ -71,7 +82,9 @@ export class UsageService {
private async initializeEventsDirectory(): Promise<void> {
if (!this.eventsDir) {
logger.warn("Stripe local file path is not configured, skipping events directory initialization.");
logger.warn(
"Stripe local file path is not configured, skipping events directory initialization."
);
return;
}
try {
@@ -83,7 +96,9 @@ export class UsageService {
private async uploadPendingEventFilesOnStartup(): Promise<void> {
if (!this.eventsDir || !this.bucketName) {
logger.warn("Stripe local file path or bucket name is not configured, skipping leftover event file upload.");
logger.warn(
"Stripe local file path or bucket name is not configured, skipping leftover event file upload."
);
return;
}
try {
@@ -106,15 +121,17 @@ export class UsageService {
ContentType: "application/json"
});
await s3Client.send(uploadCommand);
// Check if file still exists before unlinking
try {
await fs.access(filePath);
await fs.unlink(filePath);
} catch (unlinkError) {
logger.debug(`Startup file ${file} was already deleted`);
logger.debug(
`Startup file ${file} was already deleted`
);
}
logger.info(
`Uploaded leftover event file ${file} to S3 with ${events.length} events`
);
@@ -124,7 +141,9 @@ export class UsageService {
await fs.access(filePath);
await fs.unlink(filePath);
} catch (unlinkError) {
logger.debug(`Empty startup file ${file} was already deleted`);
logger.debug(
`Empty startup file ${file} was already deleted`
);
}
}
} catch (err) {
@@ -135,8 +154,8 @@ export class UsageService {
}
}
}
} catch (err) {
logger.error("Failed to scan for leftover event files:", err);
} catch (error) {
logger.error("Failed to scan for leftover event files");
}
}
@@ -146,17 +165,17 @@ export class UsageService {
value: number,
transaction: any = null
): Promise<Usage | null> {
if (build !== "saas") {
if (noop()) {
return null;
}
// Truncate value to 11 decimal places
value = this.truncateValue(value);
// Implement retry logic for deadlock handling
const maxRetries = 3;
let attempt = 0;
while (attempt <= maxRetries) {
try {
// Get subscription data for this org (with caching)
@@ -179,7 +198,12 @@ export class UsageService {
);
} else {
await db.transaction(async (trx) => {
usage = await this.internalAddUsage(orgId, featureId, value, trx);
usage = await this.internalAddUsage(
orgId,
featureId,
value,
trx
);
});
}
@@ -189,25 +213,26 @@ export class UsageService {
return usage || null;
} catch (error: any) {
// Check if this is a deadlock error
const isDeadlock = error?.code === '40P01' ||
error?.cause?.code === '40P01' ||
(error?.message && error.message.includes('deadlock'));
const isDeadlock =
error?.code === "40P01" ||
error?.cause?.code === "40P01" ||
(error?.message && error.message.includes("deadlock"));
if (isDeadlock && attempt < maxRetries) {
attempt++;
// Exponential backoff with jitter: 50-150ms, 100-300ms, 200-600ms
const baseDelay = Math.pow(2, attempt - 1) * 50;
const jitter = Math.random() * baseDelay;
const delay = baseDelay + jitter;
logger.warn(
`Deadlock detected for ${orgId}/${featureId}, retrying attempt ${attempt}/${maxRetries} after ${delay.toFixed(0)}ms`
);
await new Promise(resolve => setTimeout(resolve, delay));
await new Promise((resolve) => setTimeout(resolve, delay));
continue;
}
logger.error(
`Failed to add usage for ${orgId}/${featureId} after ${attempt} attempts:`,
error
@@ -227,10 +252,10 @@ export class UsageService {
): Promise<Usage> {
// Truncate value to 11 decimal places
value = this.truncateValue(value);
const usageId = `${orgId}-${featureId}`;
const meterId = getFeatureMeterId(featureId);
// Use upsert: insert if not exists, otherwise increment
const [returnUsage] = await trx
.insert(usage)
@@ -247,7 +272,8 @@ export class UsageService {
set: {
latestValue: sql`${usage.latestValue} + ${value}`
}
}).returning();
})
.returning();
return returnUsage;
}
@@ -268,7 +294,7 @@ export class UsageService {
value?: number,
customerId?: string
): Promise<void> {
if (build !== "saas") {
if (noop()) {
return;
}
try {
@@ -339,7 +365,7 @@ export class UsageService {
.set({
latestValue: newRunningTotal,
instantaneousValue: value,
updatedAt: Math.floor(Date.now() / 1000)
updatedAt: Math.floor(Date.now() / 1000)
})
.where(eq(usage.usageId, usageId));
}
@@ -354,7 +380,7 @@ export class UsageService {
meterId,
instantaneousValue: truncatedValue,
latestValue: truncatedValue,
updatedAt: Math.floor(Date.now() / 1000)
updatedAt: Math.floor(Date.now() / 1000)
});
}
});
@@ -415,7 +441,7 @@ export class UsageService {
): Promise<void> {
// Truncate value to 11 decimal places before sending to Stripe
const truncatedValue = this.truncateValue(value);
const event: StripeEvent = {
identifier: uuidv4(),
timestamp: Math.floor(new Date().getTime() / 1000),
@@ -432,7 +458,9 @@ export class UsageService {
private async writeEventToFile(event: StripeEvent): Promise<void> {
if (!this.eventsDir || !this.bucketName) {
logger.warn("Stripe local file path or bucket name is not configured, skipping event file write.");
logger.warn(
"Stripe local file path or bucket name is not configured, skipping event file write."
);
return;
}
if (!this.currentEventFile) {
@@ -481,7 +509,9 @@ export class UsageService {
private async uploadFileToS3(): Promise<void> {
if (!this.bucketName || !this.eventsDir) {
logger.warn("Stripe local file path or bucket name is not configured, skipping S3 upload.");
logger.warn(
"Stripe local file path or bucket name is not configured, skipping S3 upload."
);
return;
}
if (!this.currentEventFile) {
@@ -493,7 +523,9 @@ export class UsageService {
// Check if this file is already being uploaded
if (this.uploadingFiles.has(fileName)) {
logger.debug(`File ${fileName} is already being uploaded, skipping`);
logger.debug(
`File ${fileName} is already being uploaded, skipping`
);
return;
}
@@ -505,7 +537,9 @@ export class UsageService {
try {
await fs.access(filePath);
} catch (error) {
logger.debug(`File ${fileName} does not exist, may have been already processed`);
logger.debug(
`File ${fileName} does not exist, may have been already processed`
);
this.uploadingFiles.delete(fileName);
// Reset current file if it was this file
if (this.currentEventFile === fileName) {
@@ -525,7 +559,9 @@ export class UsageService {
await fs.unlink(filePath);
} catch (unlinkError) {
// File may have been already deleted
logger.debug(`File ${fileName} was already deleted during cleanup`);
logger.debug(
`File ${fileName} was already deleted during cleanup`
);
}
this.currentEventFile = null;
this.uploadingFiles.delete(fileName);
@@ -548,7 +584,9 @@ export class UsageService {
await fs.unlink(filePath);
} catch (unlinkError) {
// File may have been already deleted by another process
logger.debug(`File ${fileName} was already deleted during upload`);
logger.debug(
`File ${fileName} was already deleted during upload`
);
}
logger.info(
@@ -559,10 +597,7 @@ export class UsageService {
this.currentEventFile = null;
this.currentFileStartTime = 0;
} catch (error) {
logger.error(
`Failed to upload ${fileName} to S3:`,
error
);
logger.error(`Failed to upload ${fileName} to S3:`, error);
} finally {
// Always remove from uploading set
this.uploadingFiles.delete(fileName);
@@ -579,7 +614,7 @@ export class UsageService {
orgId: string,
featureId: FeatureId
): Promise<Usage | null> {
if (build !== "saas") {
if (noop()) {
return null;
}
@@ -598,7 +633,7 @@ export class UsageService {
`Creating new usage record for ${orgId}/${featureId}`
);
const meterId = getFeatureMeterId(featureId);
try {
const [newUsage] = await db
.insert(usage)
@@ -653,7 +688,7 @@ export class UsageService {
orgId: string,
featureId: FeatureId
): Promise<Usage | null> {
if (build !== "saas") {
if (noop()) {
return null;
}
await this.updateDaily(orgId, featureId); // Ensure daily usage is updated
@@ -673,7 +708,9 @@ export class UsageService {
*/
private async uploadOldEventFiles(): Promise<void> {
if (!this.eventsDir || !this.bucketName) {
logger.warn("Stripe local file path or bucket name is not configured, skipping old event file upload.");
logger.warn(
"Stripe local file path or bucket name is not configured, skipping old event file upload."
);
return;
}
try {
@@ -681,15 +718,17 @@ export class UsageService {
const now = Date.now();
for (const file of files) {
if (!file.endsWith(".json")) continue;
// Skip files that are already being uploaded
if (this.uploadingFiles.has(file)) {
logger.debug(`Skipping file ${file} as it's already being uploaded`);
logger.debug(
`Skipping file ${file} as it's already being uploaded`
);
continue;
}
const filePath = path.join(this.eventsDir, file);
try {
// Check if file still exists before processing
try {
@@ -704,7 +743,7 @@ export class UsageService {
if (age >= 90000) {
// 1.5 minutes - Mark as being uploaded
this.uploadingFiles.add(file);
try {
const fileContent = await fs.readFile(
filePath,
@@ -720,15 +759,17 @@ export class UsageService {
ContentType: "application/json"
});
await s3Client.send(uploadCommand);
// Check if file still exists before unlinking
try {
await fs.access(filePath);
await fs.unlink(filePath);
} catch (unlinkError) {
logger.debug(`File ${file} was already deleted during interval upload`);
logger.debug(
`File ${file} was already deleted during interval upload`
);
}
logger.info(
`Interval: Uploaded event file ${file} to S3 with ${events.length} events`
);
@@ -743,7 +784,9 @@ export class UsageService {
await fs.access(filePath);
await fs.unlink(filePath);
} catch (unlinkError) {
logger.debug(`Empty file ${file} was already deleted`);
logger.debug(
`Empty file ${file} was already deleted`
);
}
}
} finally {
@@ -765,12 +808,17 @@ export class UsageService {
}
}
public async checkLimitSet(orgId: string, kickSites = false, featureId?: FeatureId, usage?: Usage): Promise<boolean> {
if (build !== "saas") {
public async checkLimitSet(
orgId: string,
kickSites = false,
featureId?: FeatureId,
usage?: Usage
): Promise<boolean> {
if (noop()) {
return false;
}
// This method should check the current usage against the limits set for the organization
// and kick out all of the sites on the org
// and kick out all of the sites on the org
let hasExceededLimits = false;
try {
@@ -805,16 +853,30 @@ export class UsageService {
if (usage) {
currentUsage = usage;
} else {
currentUsage = await this.getUsage(orgId, limit.featureId as FeatureId);
currentUsage = await this.getUsage(
orgId,
limit.featureId as FeatureId
);
}
const usageValue = currentUsage?.instantaneousValue || currentUsage?.latestValue || 0;
logger.debug(`Current usage for org ${orgId} on feature ${limit.featureId}: ${usageValue}`);
logger.debug(`Limit for org ${orgId} on feature ${limit.featureId}: ${limit.value}`);
if (currentUsage && limit.value !== null && usageValue > limit.value) {
const usageValue =
currentUsage?.instantaneousValue ||
currentUsage?.latestValue ||
0;
logger.debug(
`Current usage for org ${orgId} on feature ${limit.featureId}: ${usageValue}`
);
logger.debug(
`Limit for org ${orgId} on feature ${limit.featureId}: ${limit.value}`
);
if (
currentUsage &&
limit.value !== null &&
usageValue > limit.value
) {
logger.debug(
`Org ${orgId} has exceeded limit for ${limit.featureId}: ` +
`${usageValue} > ${limit.value}`
`${usageValue} > ${limit.value}`
);
hasExceededLimits = true;
break; // Exit early if any limit is exceeded
@@ -823,7 +885,9 @@ export class UsageService {
// If any limits are exceeded, disconnect all sites for this organization
if (hasExceededLimits && kickSites) {
logger.warn(`Disconnecting all sites for org ${orgId} due to exceeded limits`);
logger.warn(
`Disconnecting all sites for org ${orgId} due to exceeded limits`
);
// Get all sites for this organization
const orgSites = await db
@@ -832,7 +896,7 @@ export class UsageService {
.where(eq(sites.orgId, orgId));
// Mark all sites as offline and send termination messages
const siteUpdates = orgSites.map(site => site.siteId);
const siteUpdates = orgSites.map((site) => site.siteId);
if (siteUpdates.length > 0) {
// Send termination messages to newt sites
@@ -853,17 +917,21 @@ export class UsageService {
};
// Don't await to prevent blocking
sendToClient(newt.newtId, payload).catch((error: any) => {
logger.error(
`Failed to send termination message to newt ${newt.newtId}:`,
error
);
});
sendToClient(newt.newtId, payload).catch(
(error: any) => {
logger.error(
`Failed to send termination message to newt ${newt.newtId}:`,
error
);
}
);
}
}
}
logger.info(`Disconnected ${orgSites.length} sites for org ${orgId} due to exceeded limits`);
logger.info(
`Disconnected ${orgSites.length} sites for org ${orgId} due to exceeded limits`
);
}
}
} catch (error) {

4
server/private/lib/stripe.ts
View File

@@ -14,10 +14,10 @@
import Stripe from "stripe";
import privateConfig from "#private/lib/config";
import logger from "@server/logger";
import { build } from "@server/build";
import { noop } from "@server/lib/billing/usageService";
let stripe: Stripe | undefined = undefined;
if (build == "saas") {
if (!noop()) {
const stripeApiKey = privateConfig.getRawPrivateConfig().stripe?.secret_key;
if (!stripeApiKey) {
logger.error("Stripe secret key is not configured");

5
server/private/routers/external.ts
View File

@@ -33,10 +33,11 @@ import rateLimit, { ipKeyGenerator } from "express-rate-limit";
import createHttpError from "http-errors";
import HttpCode from "@server/types/HttpCode";
import { unauthenticated as ua, authenticated as a } from "@server/routers/external";
import { unauthenticated as ua, authenticated as a, authRouter as aa } from "@server/routers/external";
export const authenticated = a;
export const unauthenticated = ua;
export const authRouter = aa;
unauthenticated.post(
"/quick-start",
@@ -227,8 +228,6 @@ authenticated.get(
loginPage.getLoginPage
);
export const authRouter = Router();
authRouter.post(
"/remoteExitNode/get-token",
rateLimit({

86
server/private/routers/hybrid.ts
View File

@@ -68,10 +68,11 @@ import { decryptData } from "@server/lib/encryption";
import config from "@server/lib/config";
import privateConfig from "#private/lib/config";
import * as fs from "fs";
import { exchangeSession } from "@server/routers/badger";
import { exchangeSession } from "@server/routers/badger";
import { validateResourceSessionToken } from "@server/auth/sessions/resource";
import { checkExitNodeOrg, resolveExitNodes } from "#private/lib/exitNodes";
import { maxmindLookup } from "@server/db/maxmind";
import { verifyResourceAccessToken } from "@server/auth/verifyResourceAccessToken";
// Zod schemas for request validation
const getResourceByDomainParamsSchema = z
@@ -162,6 +163,14 @@ const validateResourceSessionTokenBodySchema = z
})
.strict();
const validateResourceAccessTokenBodySchema = z
.object({
accessTokenId: z.string().optional(),
resourceId: z.number().optional(),
accessToken: z.string()
})
.strict();
// Certificates by domains query validation
const getCertificatesByDomainsQuerySchema = z
.object({
@@ -215,6 +224,33 @@ export type UserSessionWithUser = {
export const hybridRouter = Router();
hybridRouter.use(verifySessionRemoteExitNodeMiddleware);
hybridRouter.get(
"/general-config",
async (req: Request, res: Response, next: NextFunction) => {
return response(res, {
data: {
resource_session_request_param:
config.getRawConfig().server.resource_session_request_param,
resource_access_token_headers:
config.getRawConfig().server.resource_access_token_headers,
resource_access_token_param:
config.getRawConfig().server.resource_access_token_param,
session_cookie_name:
config.getRawConfig().server.session_cookie_name,
require_email_verification:
config.getRawConfig().flags?.require_email_verification ||
false,
resource_session_length_hours:
config.getRawConfig().server.resource_session_length_hours
},
success: true,
error: false,
message: "General config retrieved successfully",
status: HttpCode.OK
});
}
);
hybridRouter.get(
"/traefik-config",
async (req: Request, res: Response, next: NextFunction) => {
@@ -1101,6 +1137,52 @@ hybridRouter.post(
}
);
// Validate resource session token
hybridRouter.post(
"/resource/:resourceId/access-token/verify",
async (req: Request, res: Response, next: NextFunction) => {
try {
const parsedBody = validateResourceAccessTokenBodySchema.safeParse(
req.body
);
if (!parsedBody.success) {
return next(
createHttpError(
HttpCode.BAD_REQUEST,
fromError(parsedBody.error).toString()
)
);
}
const { accessToken, resourceId, accessTokenId } = parsedBody.data;
const result = await verifyResourceAccessToken({
accessTokenId,
accessToken,
resourceId
});
return response(res, {
data: result,
success: true,
error: false,
message: result.valid
? "Resource access token is valid"
: "Resource access token is invalid or expired",
status: HttpCode.OK
});
} catch (error) {
logger.error(error);
return next(
createHttpError(
HttpCode.INTERNAL_SERVER_ERROR,
"Failed to validate resource session token"
)
);
}
}
);
const geoIpLookupParamsSchema = z.object({
ip: z.string().ip()
});
@@ -1489,4 +1571,4 @@ hybridRouter.post(
);
}
}
);
);

41
server/private/routers/remoteExitNode/createRemoteExitNode.ts
View File

@@ -84,30 +84,25 @@ export async function createRemoteExitNode(
orgId,
FeatureId.REMOTE_EXIT_NODES
);
if (!usage) {
return next(
createHttpError(
HttpCode.NOT_FOUND,
"No usage data found for this organization"
)
);
}
const rejectRemoteExitNodes = await usageService.checkLimitSet(
orgId,
false,
FeatureId.REMOTE_EXIT_NODES,
{
...usage,
instantaneousValue: (usage.instantaneousValue || 0) + 1
} // We need to add one to know if we are violating the limit
);
if (rejectRemoteExitNodes) {
return next(
createHttpError(
HttpCode.FORBIDDEN,
"Remote exit node limit exceeded. Please upgrade your plan or contact us at support@fossorial.io"
)
if (usage) {
const rejectRemoteExitNodes = await usageService.checkLimitSet(
orgId,
false,
FeatureId.REMOTE_EXIT_NODES,
{
...usage,
instantaneousValue: (usage.instantaneousValue || 0) + 1
} // We need to add one to know if we are violating the limit
);
if (rejectRemoteExitNodes) {
return next(
createHttpError(
HttpCode.FORBIDDEN,
"Remote exit node limit exceeded. Please upgrade your plan or contact us at support@fossorial.io"
)
);
}
}
const secretHash = await hashPassword(secret);

2
tsconfig.json
View File

@@ -22,7 +22,7 @@
"#private/*": ["../server/private/*"],
"#open/*": ["../server/*"],
"#closed/*": ["../server/private/*"],
"#dynamic/*": ["../server/*"]
"#dynamic/*": ["../server/private/*"]
},
"plugins": [
{