-
Notifications
You must be signed in to change notification settings - Fork 9
/
pull.mjs
315 lines (226 loc) · 8.81 KB
/
pull.mjs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
// Disable ESLint console logging rule for this file
/* eslint-disable */
// Helper library written for useful postprocessing tasks with Flat Data
// Has helper functions for manipulating csv, txt, json, excel, zip, and image files
// https://droces.github.io/Deno-Cheat-Sheet/
import { writeJSON } from 'https://deno.land/x/flat@0.0.14/mod.ts'
import { exists } from "https://deno.land/std/fs/mod.ts"
import 'https://deno.land/std/dotenv/load.ts'
import { slugify } from 'https://deno.land/x/slugify/mod.ts'
import axios from 'https://deno.land/x/axiod/mod.ts'
import matter from 'https://jspm.dev/gray-matter'
import { deepmerge, deepmergeCustom } from 'https://deno.land/x/deepmergets@v4.0.3/dist/deno/mod.ts'
import {
TMDB_COMPANIES,
TMDB_LISTS,
storePath,
markdownStorePath
} from '../src/config.ts'
import { byPremiere } from '../src/helpers/sort.ts'
import {
upsertListingMarkdown,
getDataFromListingContents,
getPartsFromMarkdown,
makeTMDbMarkdownSection
} from '../src/helpers/markdown-page.ts'
import {
listingMergeConfig,
makeListingEndpoint
} from '../src/helpers/listing.ts'
// https://github.com/RebeccaStevens/deepmerge-ts/blob/beae8b841561bd206150ef02fe10db94856c6e45/docs/deepmergeCustom.md
const deepmergeListings = deepmergeCustom( listingMergeConfig )
function makeSlug ( name ) {
return slugify(name, {
lower: true,
remove: /[^a-zA-Z\d\s\-]/g,
strict: true
})
}
async function hasListingFile ( listing ) {
const listingFilePath = `${ markdownStorePath }/${ makeListingEndpoint( listing ) }.md`
return await exists( listingFilePath )
}
function cleanListings ( fetchedListings ) {
const listings = {}
for ( const id in fetchedListings ) {
// Delete noisy properties
// so we don't get a bunch of noise in our commits
delete fetchedListings[id].popularity
delete fetchedListings[id].vote_count
}
return fetchedListings
}
async function fetchListings ({
endpoint,
params = {},
listKey = 'results',
tags = []
} = {}) {
const fetchedListings = {}
let total_pages = Infinity
let page = 1
while ( page <= total_pages ) {
// console.log( 'Fetching page', company )
// console.log( 'Page', page )
// https://www.themoviedb.org/talk/5f84426469eb900039c48872
const requestUrl = `https://api.themoviedb.org${ endpoint }`
const requestOptions = {
params: {
api_key: Deno.env.get('TMDB_API_KEY'),
// language: 'en-US',
// sort_by: 'popularity.desc',
// with_companies: company.id,
page,
// Merge in params
...params
}
}
const { data } = await axios.get( requestUrl, requestOptions )
.catch( ( error ) => {
console.error( error )
})
for ( const result of data[ listKey ] ) {
// If we've already seen this title, add the new company to the list
if ( fetchedListings[ result.id ] ) {
continue
}
const title = result.name || result.title
const slug = makeSlug( title )
fetchedListings[ result.id ] = {
...result,
title,
slug,
tags,
}
}
total_pages = data.total_pages || 1
page += 1
}
return cleanListings( fetchedListings )
}
// Fetches movies from the
async function fetchListingsFromCompanies ( companies ) {
const listings = {}
for ( const company of companies ) {
const movies = await fetchListings({
endpoint: `/3/discover/movie`,
params: { with_companies: company.id },
tags: [ 'movie', `company-${ company.id }` ]
})
const tvShows = await fetchListings({
endpoint: `/3/discover/tv`,
params: { with_companies: company.id },
tags: [ 'tv', `company-${ company.id }` ]
})
// Merge into listings
for ( const id in movies ) {
listings[ id ] = deepmergeListings( movies[ id ], listings[ id ] || {} )
}
for ( const id in tvShows ) {
listings[ id ] = deepmergeListings( tvShows[ id ], listings[ id ] || {} )
}
}
// Sort movies by release_date/first_air_date and empty first
// const sortedListings = Object.values(listings)
// .sort( byListingDate )
return listings
}
async function fetchListingsFromLists ( lists ) {
const listings = {}
for ( const list of lists ) {
const fetchedListings = await fetchListings({
// https://api.themoviedb.org/3/list/8204859?append_to_response=videos,images,company&api_key={{API_KEY}}
endpoint: `/3/list/${ list.id }`,
params: {
append_to_response: 'videos,images,company'
},
tags: [ `list-${ list.id }` ],
listKey: 'items'
})
// Merge into listings
for ( const id in fetchedListings ) {
listings[ id ] = deepmergeListings( fetchedListings[ id ], listings[ id ] || {} )
}
}
return listings//sortedListings
}
// Deno specfic markdown writer
// since Deno uses custom fs and import method for gray-matter
async function writeMarkdownFileDeno ({ path, markdownBody, pageMeta }) {
// console.log('markdownBody', markdownBody)
// console.log('pageMeta', pageMeta)
const hasPageMeta = Object( pageMeta ) === pageMeta && Object.keys(pageMeta).length > 0
// Ensure markdown body end with newline
// markdownBody = markdownBody.endsWith( '\n' ) ? markdownBody : `${ markdownBody }\n`
// If there is no pageMeta
// then assume everything the body already has encoded frontmatter
if ( !hasPageMeta ) {
const bodyHasFrontmatter = markdownBody.startsWith('---')
// If there is no frontmatter then throw
if ( !bodyHasFrontmatter ) {
throw new Error(`No frontmatter found for ${ path }`)
}
await Deno.writeTextFile( path, markdownBody )
return
}
const markdownContent = matter.stringify( markdownBody, pageMeta )
// console.log('markdownContent', markdownContent)
await Deno.writeTextFile( path, markdownContent )
}
async function readMarkdownFileDeno ( filePath ) {
// const markdownContent = await fs.readFile( filePath, 'utf8' )
const markdownContent = new TextDecoder( 'utf-8' ).decode( await Deno.readFile( filePath ) )
return await getDataFromListingContents({
markdown: markdownContent,
matter
})
}
async function writeTmdbDataOnly ( listing ) {
const listingFilePath = `${ markdownStorePath }/${ makeListingEndpoint( listing ) }.md`
const decoder = new TextDecoder( 'utf-8' )
const markdownContent = decoder.decode(await Deno.readFile( listingFilePath ))
// Split off and leave behind existing TMDb data
const { existingContent } = getPartsFromMarkdown( markdownContent )
const tmdbSection = makeTMDbMarkdownSection( listing )
// console.log( 'existingContent', existingContent )
// Merge in existing meta above the current TMDb data
const content = [
existingContent.trim(),
tmdbSection,
// Ensure newline at end
// tmdbSection.endsWith( '\n' ) ? '' : '\n'
].join('\n')
await Deno.writeTextFile( listingFilePath, content )
}
async function saveListingsAsMarkdown ( listings ) {
for ( const listing of listings ) {
const listingFilePath = `${ markdownStorePath }/${ makeListingEndpoint( listing ) }.md`
if ( await exists( listingFilePath ) ) {
await writeTmdbDataOnly( listing )
continue
}
await upsertListingMarkdown( {
listing,
tmdb: listing,
readMarkdownFile: readMarkdownFileDeno,
writeMarkdownFile: writeMarkdownFileDeno,
exists: exists
})
}
}
;(async () => {
let listings = []
const companylistings = await fetchListingsFromCompanies( TMDB_COMPANIES )
const listListings = await fetchListingsFromLists( TMDB_LISTS )
// console.log('listListings', listListings[ 668 ] )
// console.log('companylistings', companylistings.find( ( listing ) => listing.id === 668 ))
listings = deepmergeListings( companylistings, listListings )
// console.log('listings', listings[ 668 ])
// Sort movies by release_date/first_air_date and empty first
const sortedListings = Object.values(listings)
.sort( byPremiere )
await writeJSON( `${ storePath }/listings.json`, sortedListings, null, '\t' )
await saveListingsAsMarkdown( sortedListings )
console.log('Pull complete.')
Deno.exit()
})()