Skip to content

Commit 961aad1

Browse files
okdargyokdargy
committed
feat: switch to web scraping + fall back on tikwm for media
1 parent a6f715a commit 961aad1

12 files changed

+3329
-116
lines changed

‎bun.lockb

37 KB
Binary file not shown.

‎package.json

+4-2
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,14 @@
22
"name": "fxtiktok",
33
"version": "1.0.0",
44
"scripts": {
5-
"dev": "wrangler dev src/index.ts",
5+
"dev": "wrangler dev src/index.ts --env=local",
66
"build": "wrangler build src/index.ts",
77
"deploy": "wrangler deploy --minify src/index.ts"
88
},
99
"dependencies": {
10-
"hono": "^3.11.7"
10+
"@types/set-cookie-parser": "^2.4.7",
11+
"hono": "^3.11.7",
12+
"set-cookie-parser": "^2.6.0"
1113
},
1214
"devDependencies": {
1315
"@cloudflare/workers-types": "^4.20230914.0",

‎src/index.ts

+47-28
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,19 @@
11
import { Hono } from 'hono'
22
import { cache } from 'hono/cache'
33

4-
import { grabAwemeId, getVideoInfo } from './services/tiktok'
4+
import { scrapeVideoData } from './services/tiktok'
5+
import { grabAwemeId } from './services/tiktok'
56
import { VideoResponse, ErrorResponse } from './templates'
67
import generateAlternate from './util/generateAlternate'
7-
import { returnHTMLResponse } from './util/ResponseHelper'
8+
import { returnHTMLResponse } from './util/responseHelper'
9+
10+
import { ItemStruct } from './types/Web'
811

912
const app = new Hono()
1013

1114
app.get('/test/:videoId', async (c) => {
1215
const { videoId } = c.req.param()
13-
const awemeId = await getVideoInfo(videoId)
16+
const awemeId = await scrapeVideoData(videoId)
1417

1518
if(awemeId instanceof Error) {
1619
return new Response((awemeId as Error).message, { status: 500 })
@@ -62,28 +65,27 @@ async function handleVideo(c: any): Promise<Response> {
6265
}
6366

6467
try {
65-
const videoInfo = await getVideoInfo(id)
68+
const videoInfo = await scrapeVideoData(id)
6669

6770
if (videoInfo instanceof Error) {
6871
const responseContent = await ErrorResponse((videoInfo as Error).message);
6972
return returnHTMLResponse(responseContent, 201);
7073
}
7174

7275
const url = new URL(c.req.url);
73-
7476
if(url.hostname.includes('d.tnktok.com') || c.req.query('isDirect') === 'true') {
7577
if(videoInfo.video.duration > 0) {
7678
return new Response('', {
7779
status: 302,
7880
headers: {
79-
'Location': 'https://fxtiktok-rewrite.dargy.workers.dev/generate/video/' + videoInfo.aweme_id
81+
'Location': 'https://fxtiktok-rewrite.dargy.workers.dev/generate/video/' + videoInfo.id
8082
}
8183
})
8284
} else {
8385
return new Response('', {
8486
status: 302,
8587
headers: {
86-
'Location': 'https://fxtiktok-rewrite.dargy.workers.dev/generate/image/' + videoInfo.aweme_id
88+
'Location': 'https://fxtiktok-rewrite.dargy.workers.dev/generate/image/' + videoInfo.id
8789
}
8890
})
8991
}
@@ -92,6 +94,7 @@ async function handleVideo(c: any): Promise<Response> {
9294
return returnHTMLResponse(responseContent);
9395
}
9496
} catch(e) {
97+
console.log(e);
9598
const responseContent = await ErrorResponse((e as Error).message);
9699
return returnHTMLResponse(responseContent, 201);
97100
}
@@ -118,20 +121,26 @@ app.get(
118121

119122
app.get('/generate/video/:videoId', async (c) => {
120123
const { videoId } = c.req.param()
121-
const data = await getVideoInfo(videoId);
122124

123-
if (data instanceof Error) {
124-
return new Response((data as Error).message, { status: 500,
125-
headers: {
126-
'Cache-Control': 'no-cache, no-store, must-revalidate',
125+
try {
126+
/*
127+
const data = await scrapeVideoData(videoId);
128+
129+
if (!(data instanceof Error)) {
130+
if(data.video.playAddr) {
131+
return c.redirect(data.video.playAddr)
132+
} else {
133+
return new Response('No video found', { status: 404,
134+
headers: {
135+
'Cache-Control': 'no-cache, no-store, must-revalidate',
136+
}
137+
})
127138
}
128-
})
129-
}
130-
131-
if(data.video.play_addr.url_list.length > 0) {
132-
return c.redirect(data.video.play_addr.url_list[0])
133-
} else {
134-
return new Response('No video found', { status: 404,
139+
}
140+
*/
141+
return c.redirect(`https://tikwm.com/video/media/play/${videoId}.mp4`);
142+
} catch(e) {
143+
return new Response((e as Error).message, { status: 500,
135144
headers: {
136145
'Cache-Control': 'no-cache, no-store, must-revalidate',
137146
}
@@ -141,16 +150,26 @@ app.get('/generate/video/:videoId', async (c) => {
141150

142151
app.get('/generate/image/:videoId', async (c) => {
143152
const { videoId } = c.req.param()
144-
const data = await getVideoInfo(videoId);
145153

146-
if (data instanceof Error) {
147-
return new Response((data as Error).message, { status: 500 })
148-
}
149-
150-
if(data.video.cover.url_list.length > 0) {
151-
return c.redirect(data.video.cover.url_list[0])
152-
} else {
153-
return new Response(JSON.stringify(data), { status: 200 })
154+
try {
155+
/*
156+
const data = await scrapeVideoData(videoId);
157+
158+
if (!(data instanceof Error)) {
159+
if(data.imagePost.images.length > 0) {
160+
return c.redirect(data.imagePost.images[0].imageURL.urlList[0])
161+
} else {
162+
return new Response(JSON.stringify(data), { status: 200 })
163+
}
164+
}
165+
*/
166+
return c.redirect(`https://tikwm.com/video/cover/${videoId}.webp`);
167+
} catch(e) {
168+
return new Response((e as Error).message, { status: 500,
169+
headers: {
170+
'Cache-Control': 'no-cache, no-store, must-revalidate',
171+
}
172+
})
154173
}
155174
})
156175

‎src/services/tiktok.ts

+60-59
Original file line numberDiff line numberDiff line change
@@ -1,63 +1,64 @@
1-
import { TikTokAPIResponse, AwemeList } from "../types/Services";
1+
import { WebJSONResponse, ItemStruct } from "../types/Web";
2+
import Cookie from "../util/cookieHelper";
3+
import cookieParser from "set-cookie-parser";
4+
5+
const cookie = new Cookie([]);
26

37
export async function grabAwemeId(videoId: string): Promise<String | Error> {
4-
// https://vm.tiktok.com/ZMJmVWVpL/
5-
const res = await fetch("https://vm.tiktok.com/" + videoId);
6-
const url = new URL(res.url);
7-
8-
const awemeIdPattern = /\/@[\w\d_.]+\/(video|photo)\/(\d{19})/;
9-
const match = url.pathname.match(awemeIdPattern);
10-
11-
if (match) {
12-
return match[2];
13-
} else {
14-
throw new Error("Could not find awemeId");
15-
}
16-
}
8+
// https://vm.tiktok.com/ZMJmVWVpL/
9+
const res = await fetch("https://vm.tiktok.com/" + videoId);
10+
const url = new URL(res.url);
11+
12+
const awemeIdPattern = /\/@[\w\d_.]+\/(video|photo)\/(\d{19})/;
13+
const match = url.pathname.match(awemeIdPattern);
1714

18-
export async function getVideoInfo(
19-
awemeId: String,
20-
): Promise<AwemeList | Error> {
21-
const apiUrl = new URL(
22-
"https://api22-normal-c-alisg.tiktokv.com/aweme/v1/feed/?region=US&carrier_region=US",
23-
);
24-
25-
const params = {
26-
aweme_id: awemeId,
27-
iid: "7318518857994389254",
28-
device_id: "7318517321748022790",
29-
channel: "googleplay",
30-
app_name: "musical_ly",
31-
version_code: "300904",
32-
device_platform: "android",
33-
device_type: "ASUS_Z01QD",
34-
os_version: "9",
35-
};
36-
37-
Object.keys(params).forEach((key) =>
38-
apiUrl.searchParams.append(key, params[key]),
39-
);
40-
41-
console.log(apiUrl.toString());
42-
43-
const res: Response = await fetch(apiUrl.toString(), {
44-
headers: {
45-
"User-Agent":
46-
"Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Mobile Safari/537.36",
47-
},
48-
cf: {
49-
cacheEverything: true,
50-
cacheTtlByStatus: { "200-299": 86400, 404: 1, "500-599": 0 },
51-
},
52-
});
53-
const json: TikTokAPIResponse = await res.json();
54-
const videoInfo: AwemeList | undefined = json.aweme_list.find(
55-
(aweme) => aweme.aweme_id === awemeId,
56-
);
57-
58-
if (videoInfo) {
59-
return videoInfo;
60-
} else {
61-
return new Error("Could not find video info");
62-
}
15+
if (match) {
16+
return match[2];
17+
} else {
18+
throw new Error("Could not find awemeId");
19+
}
6320
}
21+
22+
export async function scrapeVideoData(
23+
awemeId: string,
24+
author?: string
25+
): Promise<ItemStruct | Error> {
26+
console.log('before', cookie.getUpdatingCookies());
27+
const res = await fetch(`https://www.tiktok.com/@${author || "i"}"/video/${awemeId}`, {
28+
method: "GET",
29+
headers: {
30+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
31+
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0",
32+
"Cookie": cookie.getCookiesAsString(),
33+
},
34+
cf: {
35+
cacheEverything: true,
36+
cacheTtlByStatus: { "200-299": 86400, 404: 1, "500-599": 0 },
37+
},
38+
});
39+
40+
console.log('string', cookie.getCookiesAsString());
41+
console.log(res.headers)
42+
let cookies = cookieParser(res.headers.get("set-cookie")!);
43+
cookie.setCookies(cookies);
44+
45+
const html = await res.text();
46+
47+
try {
48+
const resJson = html.split('<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">')[1].split('</script>')[0]
49+
const json: WebJSONResponse = JSON.parse(resJson);
50+
51+
//console.log(Object.keys(json["__DEFAULT_SCOPE__"]));
52+
if(!json["__DEFAULT_SCOPE__"]["webapp.video-detail"] || json["__DEFAULT_SCOPE__"]["webapp.video-detail"].statusCode == 10204) throw new Error("Could not find video data");
53+
const videoInfo = json["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"];
54+
//console.log(videoInfo)
55+
56+
return videoInfo
57+
} catch(err) {
58+
console.log(err);
59+
throw new Error("Could not parse video info");
60+
}
61+
62+
63+
64+
}

‎src/services/tiktokv.ts

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
import { TikTokAPIResponse, AwemeList } from "../types/API";
2+
3+
export async function getVideoInfo(
4+
awemeId: string,
5+
): Promise<AwemeList | Error> {
6+
const apiUrl = new URL(
7+
"https://api22-normal-c-alisg.tiktokv.com/aweme/v1/feed/",
8+
);
9+
10+
apiUrl.search = new URLSearchParams({
11+
region: "US",
12+
carrier_region: "US",
13+
aweme_id: awemeId,
14+
iid: "7318518857994389254",
15+
device_id: "7318517321748022790",
16+
channel: "googleplay",
17+
app_name: "musical_ly",
18+
version_code: "300904",
19+
device_platform: "android",
20+
device_type: "ASUS_Z01QD",
21+
os_version: "9",
22+
}).toString();
23+
24+
const res: Response = await fetch(apiUrl.toString(), {
25+
headers: {
26+
"User-Agent":
27+
"Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Mobile Safari/537.36",
28+
},
29+
cf: {
30+
cacheEverything: true,
31+
cacheTtlByStatus: { "200-299": 86400, 404: 1, "500-599": 0 },
32+
},
33+
});
34+
const json: TikTokAPIResponse = await res.json();
35+
const videoInfo: AwemeList | undefined = json.aweme_list.find(
36+
(aweme) => aweme.aweme_id === awemeId,
37+
);
38+
39+
if (videoInfo) {
40+
return videoInfo;
41+
} else {
42+
return new Error("Could not find video info");
43+
}
44+
}

‎src/templates/pages/Error.tsx

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import { AwemeList } from '../../types/Services';
2-
import MetaHelper from '../../util/MetaHelper';
1+
import { AwemeList } from '../../types/API';
2+
import MetaHelper from '../../util/metaHelper';
33

44
export function ErrorResponse(error: string): JSX.Element {
55
return (

0 commit comments

Comments
 (0)