Merge remote-tracking branch 'origin/newjitsu' into newjitsu

.github/workflows/close-stale-issues.yml

@@ -0,0 +1,23 @@
+name: Close inactive issues
+on:
+  schedule:
+    - cron: "30 1 * * *" #once a day at 1:30am
+
+jobs:
+  close-issues:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v5
+        with:
+          days-before-issue-stale: 15
+          days-before-issue-close: 15
+          stale-issue-label: "🕰️Stale"
+          exempt-issue-labels: "⏳Postpone"
+          stale-issue-message: "This issue is stale because it has been open for 30 days with no activity."
+          close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale."
+          days-before-pr-stale: -1
+          days-before-pr-close: -1
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

README.md

@@ -2,7 +2,7 @@
 
 <p align="center">
 👉<b>Looking for Jitsu Classic? Switch to 
-<a href="https://github.com/jitsucom/jitsu/tree/master">classic branch</a>, and read about <a href="https://docs.jitsu.com/jitsu-classic">Jitsu Classic and Jitsu Next differences</a></b>
+<a href="https://github.com/jitsucom/jitsu/tree/master">classic branch</a>, and read about <a href="[https://docs.jitsu.com/jitsu-classic](https://jitsu.com/blog/jitsu-next">Jitsu Classic and Jitsu Next differences</a></b>
 </p>
 <p align="center">
 <img src="https://github.com/jitsucom/jitsu/blob/newjitsu/.readme-assets/github-hero-light-mode.png?raw=true#gh-light-mode-only" />

webapps/console/components/ServicesCatalog/ServicesCatalog.tsx

@@ -8,7 +8,7 @@ import { LoadingAnimation } from "../GlobalLoader/GlobalLoader";
 import React from "react";
 import { ErrorCard } from "../GlobalError/GlobalError";
 import { Input } from "antd";
-import { useAppConfig } from "../../lib/context";
+import { useAppConfig, useWorkspace } from "../../lib/context";
 
 function groupByType(sources: SourceType[]): Record<string, SourceType[]> {
   const groups: Record<string, SourceType[]> = {};
@@ -57,6 +57,7 @@ export function getServiceIcon(source: SourceType, icons: Record<string, string>
 export const ServicesCatalog: React.FC<{ onClick: (packageType, packageId: string) => void }> = ({ onClick }) => {
   const { data, isLoading, error } = useApi<{ sources: SourceType[] }>(`/api/sources?mode=meta`);
   const sourcesIconsLoader = useApi<{ sources: SourceType[] }>(`/api/sources?mode=icons-only`);
+  const workspace = useWorkspace();
   const [filter, setFilter] = React.useState("");
   const appconfig = useAppConfig();
   const sourcesIcons: Record<string, string> = sourcesIconsLoader.data
@@ -95,6 +96,7 @@ export const ServicesCatalog: React.FC<{ onClick: (packageType, packageId: strin
             .filter(
               source =>
                 !appconfig.mitCompliant ||
+                workspace.featuresEnabled.includes("ignore_sources_licenses") ||
                 source.meta.license?.toLowerCase() === "mit" ||
                 (source.meta.mitVersions && source.meta.mitVersions.length > 0)
             );

webapps/console/lib/api.ts

@@ -76,6 +76,9 @@ function parseIfNeeded(o: any): any {
 export function getAuthBearerToken(req: NextApiRequest): string | undefined {
   if (req.headers.authorization && req.headers.authorization.toLowerCase().indexOf("bearer ") === 0) {
     return req.headers.authorization.substring("bearer ".length);
+  } else if (req.query?.__unsafe_token) {
+    //very unsafe, but some tools we use can't set headers, so we need to allow this
+    return req.query.__unsafe_token as string;
   }
   return undefined;
 }

webapps/console/lib/ee-client.ts

@@ -1,5 +1,4 @@
 import { get } from "./useApi";
-import { DomainStatus } from "./server/ee";
 import * as auth from "firebase/auth";
 
 export type ClassicProjectStatus = {
@@ -11,7 +10,6 @@ export type ClassicProjectStatus = {
 };
 
 export interface EeClient {
-  attachDomain(domain: string): Promise<DomainStatus>;
   checkClassicProject(): Promise<ClassicProjectStatus>;
   createCustomToken(): Promise<string>;
 }
@@ -39,15 +37,6 @@ export function getEeClient(host: string, workspaceId: string): EeClient {
     return cachedToken;
   };
   return {
-    attachDomain: async domain => {
-      cachedToken = await refreshTokenIfNeeded();
-      return await get(removeDoubleSlashes(`${host}/api/domain`), {
-        query: { domain },
-        headers: {
-          Authorization: `Bearer ${cachedToken.token}`,
-        },
-      });
-    },
     checkClassicProject: async () => {
       const fbToken = await auth.getAuth().currentUser?.getIdToken();
       return await get(removeDoubleSlashes(`${host}/api/is-active`), {

webapps/console/lib/schema/index.ts

@@ -56,6 +56,7 @@ export const AppConfig = z.object({
   //iso date
   readOnlyUntil: z.string().optional(),
   disableSignup: z.boolean().optional(),
+  customDomainsEnabled: z.boolean().optional(),
   ee: z.object({
     available: z.boolean(),
     host: z.string().optional(),

webapps/console/lib/server/custom-domains.ts

@@ -1,8 +1,12 @@
 import { db } from "./db";
 import { StreamConfig } from "../schema";
+import dns from "dns";
+import { getLog } from "juava";
 
 type DomainAvailability = { available: true; usedInWorkspaces?: never } | { available: false; usedInWorkspace: string };
 
+export const customDomainCnames = process.env.CUSTOM_DOMAIN_CNAMES?.split(",");
+
 /**
  * Tells if the given domain is used in other workspaces.
  */
@@ -26,3 +30,38 @@ export async function isDomainAvailable(domain: string, workspaceId: string): Pr
     return { available: true };
   }
 }
+
+function resolveCname(domain: string): Promise<string | undefined> {
+  return new Promise((resolve, reject) => {
+    dns.resolveCname(domain, (err, addresses) => {
+      if (err) {
+        reject(err);
+      } else {
+        if (addresses.length === 1) {
+          resolve(addresses[0]);
+        } else if (!addresses || addresses.length === 0) {
+          resolve(undefined);
+        } else {
+          getLog()
+            .atWarn()
+            .log(`Domain ${domain} has multiple CNAME records: ${addresses.join(", ")}. Using first one`);
+          resolve(addresses[0]);
+        }
+      }
+    });
+  });
+}
+
+export async function isCnameValid(domain: string): Promise<boolean> {
+  if (!customDomainCnames || customDomainCnames.length == 0) {
+    throw new Error(`CUSTOM_DOMAIN_CNAMES is not set. isCnameValid() should not be called`);
+  }
+  let cnameRecord: string | undefined;
+  try {
+    cnameRecord = await resolveCname(domain);
+  } catch (e) {
+    getLog().atError().withCause(e).log(`Domain ${domain} has no CNAME records`);
+    return false;
+  }
+  return !!(cnameRecord && customDomainCnames.includes(cnameRecord.toLowerCase()));
+}

webapps/console/lib/server/ee.ts

@@ -45,13 +45,3 @@ export function createJwt(
   const token = jwt.sign({ userId, email, workspaceId, exp: expiresSecondsTimestamp }, jwtSecret);
   return { jwt: token, expiresAt: new Date(expiresSecondsTimestamp * 1000).toISOString() };
 }
-
-export type DomainStatus = { error?: string } & (
-  | { needsConfiguration: false }
-  | { needsConfiguration: true; configurationType: "cname"; cnameValue: string }
-  | {
-      needsConfiguration: true;
-      configurationType: "verification";
-      verification: { type: string; domain: string; value: string }[];
-    }
-);

webapps/console/lib/shared/domain-check-response.ts

@@ -0,0 +1,21 @@
+import { z } from "zod";
+import { Simplify } from "type-fest";
+
+export const DomainCheckResponse = z.union([
+  z.object({
+    ok: z.literal(true),
+    reason: z.never().optional(),
+  }),
+  z.object({
+    ok: z.literal(false),
+    reason: z.union([z.literal("used_by_other_workspace"), z.literal("invalid_domain_name")]),
+    cnameValue: z.never().optional(),
+  }),
+  z.object({
+    ok: z.literal(false),
+    reason: z.literal("requires_cname_configuration"),
+    cnameValue: z.string().optional(),
+  }),
+]);
+
+export type DomainCheckResponse = Simplify<z.infer<typeof DomainCheckResponse>>;

webapps/console/pages/[workspaceId]/services.tsx

@@ -155,7 +155,11 @@ const ServicesList: React.FC<{}> = () => {
         `/api/sources/versions?type=${packageType}&package=${encodeURIComponent(packageId)}`
       );
       const versions = rawVersions.versions
-        .filter((v: any) => v.isRelease && (v.isMit || !appconfig.mitCompliant))
+        .filter(
+          (v: any) =>
+            v.isRelease &&
+            (v.isMit || !appconfig.mitCompliant || workspace.featuresEnabled.includes("ignore_sources_licenses"))
+        )
         .map((v: any) => v.name);
       const sourceType = await rpc(`/api/sources/${packageType}/${encodeURIComponent(packageId)}`);
 

webapps/console/pages/[workspaceId]/streams.tsx

@@ -15,7 +15,6 @@ import { getEeClient } from "../../lib/ee-client";
 import { assertDefined, requireDefined } from "juava";
 import { ReloadOutlined } from "@ant-design/icons";
 import { confirmOp, feedbackError } from "../../lib/ui";
-import type { DomainStatus } from "../../lib/server/ee";
 import { getAntdModal, useAntdModal } from "../../lib/modal";
 import { get } from "../../lib/useApi";
 import { Activity, AlertTriangle, Check, Globe, Wrench, Zap } from "lucide-react";
@@ -27,6 +26,7 @@ import { useLinksQuery } from "../../lib/queries";
 import { toURL } from "../../lib/shared/url";
 import JSON5 from "json5";
 import { EditorToolbar } from "../../components/EditorToolbar/EditorToolbar";
+import { DomainCheckResponse } from "../../lib/shared/domain-check-response";
 
 const Streams: React.FC<any> = () => {
   return (
@@ -77,10 +77,10 @@ const CustomDomain: React.FC<{ domain: string; deleteDomain: () => Promise<void>
   );
   const [reloadTrigger, setReloadTrigger] = useState(0);
   const [deleting, setDeleting] = useState(false);
-  const { data, isLoading, error, refetch } = useQuery<DomainStatus>(
+  const { data, isLoading, error, refetch } = useQuery<DomainCheckResponse>(
     ["domain-status", domain.toLowerCase(), reloadTrigger],
     async () => {
-      return await eeClient.attachDomain(domain);
+      return await get(`/api/${workspace.id}/domain-check?domain=${domain.toLowerCase()}`);
     },
     { cacheTime: 0 }
   );
@@ -94,9 +94,7 @@ const CustomDomain: React.FC<{ domain: string; deleteDomain: () => Promise<void>
           {/*</div>*/}
           <div className={"text-blue-600 w-4 h-4 mr-1.5"}>
             <Globe
-              className={`w-full h-full ${
-                error || data?.error ? "text-red-600" : data?.needsConfiguration ? "text-yellow-600" : "text-blue-600"
-              }`}
+              className={`w-full h-full ${error ? "text-red-600" : data?.ok ? "text-blue-600" : "text-yellow-600"}`}
             />
           </div>
           <div className="font-bold  text-lg">{domain}</div>
@@ -113,14 +111,14 @@ const CustomDomain: React.FC<{ domain: string; deleteDomain: () => Promise<void>
                 <FaExternalLinkAlt />
               </Button>
             </Tooltip>
-            {data?.needsConfiguration && (
+            {!data?.ok && (
               <Tooltip title="See configuration instructions">
                 <Button
                   type="text"
                   danger
                   disabled={isLoading || deleting}
                   onClick={() => {
-                    DomainConfigurationInstructions.show({ domain, status: data });
+                    DomainConfigurationInstructions.show({ domain, status: data! });
                   }}
                   className="border-0"
                 >
@@ -174,29 +172,29 @@ const CustomDomain: React.FC<{ domain: string; deleteDomain: () => Promise<void>
                   </span>
                 </StatusBadge>
               );
-            } else if (error || data?.error) {
+            } else if (error) {
               return <StatusBadge status="error">ERROR</StatusBadge>;
-            } else if (data?.needsConfiguration) {
+            } else if (!data?.ok) {
               return <StatusBadge status="warning">Configuration Required</StatusBadge>;
             } else {
               return <StatusBadge status="success">OK</StatusBadge>;
             }
           })()}
         </div>
-        {(error || data?.error) && (
+        {error && (
           <div className="flex items-start mt-1">
             <div className={"mr-2"}>Description:</div>
-            <div className="">{`${data?.error || "Internal error"}`}</div>
+            <div className="">{`${"Internal error"}`}</div>
           </div>
         )}
-        {data?.needsConfiguration && (
+        {!data?.ok && (
           <div className="flex items-start mt-1">
             <div className={"mr-2"}>Description:</div>
             <div className="">
               See{" "}
               <a
                 className={"cursor-pointer"}
-                onClick={() => DomainConfigurationInstructions.show({ domain, status: data })}
+                onClick={() => DomainConfigurationInstructions.show({ domain, status: data! })}
               >
                 <u>configuration instructions</u>
               </a>
@@ -234,25 +232,16 @@ export const DNSRecordTable: React.FC<DNSRecordTableProps> = ({ records }) => {
   );
 };
 
-export type DomainInstructionsProps = { domain: string; status: DomainStatus };
+export type DomainInstructionsProps = { domain: string; status: DomainCheckResponse };
 const DomainConfigurationInstructions: React.FC<DomainInstructionsProps> & {
   show: (p: DomainInstructionsProps) => void;
 } = ({ domain, status }) => {
-  if (status.needsConfiguration && status.configurationType === "cname") {
+  if (status.reason === "requires_cname_configuration") {
     return (
       <div>
         <h3>Set the following record on your DNS provider to continue</h3>
         <p className="bg-bgLight py-2 my-4">
-          <DNSRecordTable records={[{ type: "CNAME", domain, value: status.cnameValue }]} />
-        </p>
-      </div>
-    );
-  } else if (status.needsConfiguration && status.configurationType == "verification") {
-    return (
-      <div>
-        <h3>Set the following record on your DNS provider to continue</h3>
-        <p className="bg-bgLight py-2 my-4">
-          <DNSRecordTable records={status.verification} />
+          <DNSRecordTable records={[{ type: "CNAME", domain, value: status.cnameValue! }]} />
         </p>
       </div>
     );
@@ -282,10 +271,24 @@ const DomainsEditor: React.FC<CustomWidgetProps<string[]>> = props => {
   const add = async () => {
     setAddPending(true);
     try {
-      const { available } = await get(`/api/${workspace.id}/domain-check?domain=${addValue}`);
-      if (!available) {
-        feedbackError(`Domain ${addValue} is not available. It is used by other workspace`);
-        return;
+      const available: DomainCheckResponse = await get(`/api/${workspace.id}/domain-check?domain=${addValue}`);
+      if (!available.ok) {
+        if (available.reason === "used_by_other_workspace") {
+          feedbackError(
+            <>
+              Domain <code>{addValue}</code> is not available. It is used by other workspace. Contact{" "}
+              <code>[email protected]</code> if you think this is a mistake
+            </>
+          );
+          return;
+        } else if (available.reason === "invalid_domain_name") {
+          feedbackError(
+            <>
+              Invalid domain name <code>{addValue}</code>
+            </>
+          );
+          return;
+        }
       }
       const newVal = [...domains, addValue as string];
       setDomains(newVal);
@@ -544,7 +547,7 @@ const StreamsList: React.FC<{}> = () => {
       },
       domains: {
         editor: DomainsEditor,
-        hidden: !appConfig.ee.available,
+        hidden: !appConfig.customDomainsEnabled,
         displayName: "Custom Tracking Domains",
         documentation: (
           <>