mirror of
https://github.com/iptv-org/iptv.git
synced 2024-12-26 17:37:02 +00:00
Update scripts
This commit is contained in:
parent
89b87ad5c8
commit
bb2935878d
|
@ -1,16 +1,16 @@
|
|||
import { API_DIR, DB_DIR } from '../../constants'
|
||||
import { Logger, Database, Collection, Storage } from '../../core'
|
||||
import { API_DIR, STREAMS_DIR } from '../../constants'
|
||||
import { Logger, PlaylistParser, Storage } from '../../core'
|
||||
import { Stream } from '../../models'
|
||||
|
||||
async function main() {
|
||||
const logger = new Logger()
|
||||
|
||||
logger.info(`loading streams...`)
|
||||
const db = new Database(DB_DIR)
|
||||
const dbStreams = await db.load('streams.db')
|
||||
const docs = await dbStreams.find({})
|
||||
|
||||
const streams = new Collection(docs as any[])
|
||||
logger.info('loading streams...')
|
||||
const streamsStorage = new Storage(STREAMS_DIR)
|
||||
const parser = new PlaylistParser({ storage: streamsStorage })
|
||||
const files = await streamsStorage.list('**/*.m3u')
|
||||
let streams = await parser.parse(files)
|
||||
streams = streams
|
||||
.map(data => new Stream(data))
|
||||
.orderBy((stream: Stream) => stream.channel)
|
||||
.map((stream: Stream) => stream.toJSON())
|
||||
|
@ -18,8 +18,8 @@ async function main() {
|
|||
logger.info(`found ${streams.count()} streams`)
|
||||
|
||||
logger.info('saving to .api/streams.json...')
|
||||
const storage = new Storage(API_DIR)
|
||||
await storage.save('streams.json', streams.toJSON())
|
||||
const apiStorage = new Storage(API_DIR)
|
||||
await apiStorage.save('streams.json', streams.toJSON())
|
||||
}
|
||||
|
||||
main()
|
||||
|
|
|
@ -4,7 +4,6 @@ mkdir -p temp/data
|
|||
curl -L -o temp/data/blocklist.json https://iptv-org.github.io/api/blocklist.json
|
||||
curl -L -o temp/data/categories.json https://iptv-org.github.io/api/categories.json
|
||||
curl -L -o temp/data/channels.json https://iptv-org.github.io/api/channels.json
|
||||
curl -L -o temp/data/streams.json https://iptv-org.github.io/api/streams.json
|
||||
curl -L -o temp/data/countries.json https://iptv-org.github.io/api/countries.json
|
||||
curl -L -o temp/data/languages.json https://iptv-org.github.io/api/languages.json
|
||||
curl -L -o temp/data/regions.json https://iptv-org.github.io/api/regions.json
|
||||
|
|
|
@ -1,33 +0,0 @@
|
|||
import { Storage, Logger, PlaylistParser, Collection, Database } from '../../core'
|
||||
import { Stream, Playlist } from '../../models'
|
||||
import { STREAMS_DIR, DB_DIR } from '../../constants'
|
||||
|
||||
async function main() {
|
||||
const logger = new Logger()
|
||||
|
||||
logger.info(`looking for streams...`)
|
||||
const storage = new Storage(STREAMS_DIR)
|
||||
const parser = new PlaylistParser({
|
||||
storage
|
||||
})
|
||||
const files = await storage.list(`**/*.m3u`)
|
||||
let streams = new Collection()
|
||||
for (let filepath of files) {
|
||||
const playlist: Playlist = await parser.parse(filepath)
|
||||
streams = streams.concat(playlist.streams)
|
||||
}
|
||||
|
||||
logger.info(`found ${streams.count()} streams`)
|
||||
|
||||
logger.info('clean up the storage...')
|
||||
const dbStorage = new Storage(DB_DIR)
|
||||
await dbStorage.clear('streams.db')
|
||||
|
||||
logger.info('saving streams to the database...')
|
||||
const db = new Database(DB_DIR)
|
||||
const dbStreams = await db.load('streams.db')
|
||||
const data = streams.map((stream: Stream) => stream.data()).all()
|
||||
await dbStreams.insert(data)
|
||||
}
|
||||
|
||||
main()
|
67
scripts/commands/playlist/format.ts
Normal file
67
scripts/commands/playlist/format.ts
Normal file
|
@ -0,0 +1,67 @@
|
|||
import { STREAMS_DIR, DATA_DIR } from '../../constants'
|
||||
import { Storage, Logger, PlaylistParser, Collection } from '../../core'
|
||||
import { Stream, Playlist, Channel } from '../../models'
|
||||
import { program } from 'commander'
|
||||
|
||||
program.argument('[filepath]', 'Path to file to validate').parse(process.argv)
|
||||
|
||||
async function main() {
|
||||
const storage = new Storage(STREAMS_DIR)
|
||||
const logger = new Logger()
|
||||
|
||||
logger.info('loading channels from api...')
|
||||
const dataStorage = new Storage(DATA_DIR)
|
||||
const channelsContent = await dataStorage.json('channels.json')
|
||||
const groupedChannels = new Collection(channelsContent)
|
||||
.map(data => new Channel(data))
|
||||
.keyBy((channel: Channel) => channel.id)
|
||||
|
||||
logger.info('loading streams...')
|
||||
const parser = new PlaylistParser({ storage })
|
||||
const files = program.args.length ? program.args : await storage.list('**/*.m3u')
|
||||
let streams = await parser.parse(files)
|
||||
|
||||
logger.info(`found ${streams.count()} streams`)
|
||||
|
||||
logger.info('normalizing links...')
|
||||
streams = streams.map(stream => {
|
||||
stream.normalizeURL()
|
||||
return stream
|
||||
})
|
||||
|
||||
logger.info('removing duplicates...')
|
||||
streams = streams.uniqBy(stream => stream.url)
|
||||
|
||||
logger.info('removing wrong id...')
|
||||
streams = streams.map((stream: Stream) => {
|
||||
if (groupedChannels.missing(stream.channel)) {
|
||||
stream.channel = ''
|
||||
}
|
||||
|
||||
return stream
|
||||
})
|
||||
|
||||
logger.info('sorting links...')
|
||||
streams = streams.orderBy(
|
||||
[
|
||||
(stream: Stream) => stream.name,
|
||||
(stream: Stream) => parseInt(stream.quality.replace('p', '')),
|
||||
(stream: Stream) => stream.label,
|
||||
(stream: Stream) => stream.url
|
||||
],
|
||||
['asc', 'desc', 'asc', 'asc']
|
||||
)
|
||||
|
||||
logger.info('saving...')
|
||||
const groupedStreams = streams.groupBy((stream: Stream) => stream.filepath)
|
||||
for (let filepath of groupedStreams.keys()) {
|
||||
const streams = groupedStreams.get(filepath) || []
|
||||
|
||||
if (!streams.length) return
|
||||
|
||||
const playlist = new Playlist(streams, { public: false })
|
||||
await storage.save(filepath, playlist.toString())
|
||||
}
|
||||
}
|
||||
|
||||
main()
|
|
@ -1,6 +1,5 @@
|
|||
import { File, Storage } from '../../core'
|
||||
import { File, PlaylistParser, Storage } from '../../core'
|
||||
import { Stream, Category, Channel, Language, Country, Region, Subdivision } from '../../models'
|
||||
import { Database } from '../../core/database'
|
||||
import { Collection } from '../../core/collection'
|
||||
import { Logger } from '../../core/logger'
|
||||
import _ from 'lodash'
|
||||
|
@ -16,32 +15,29 @@ import {
|
|||
IndexLanguageGenerator,
|
||||
IndexRegionGenerator
|
||||
} from '../../generators'
|
||||
import { DATA_DIR, DB_DIR, LOGS_DIR } from '../../constants'
|
||||
import { DATA_DIR, LOGS_DIR, STREAMS_DIR } from '../../constants'
|
||||
|
||||
async function main() {
|
||||
const logger = new Logger()
|
||||
const dataStorage = new Storage(DATA_DIR)
|
||||
|
||||
const storage = new Storage(DATA_DIR)
|
||||
|
||||
const channelsContent = await storage.json('channels.json')
|
||||
logger.info('loading data from api...')
|
||||
const channelsContent = await dataStorage.json('channels.json')
|
||||
const channels = new Collection(channelsContent).map(data => new Channel(data))
|
||||
|
||||
const categoriesContent = await storage.json('categories.json')
|
||||
const categoriesContent = await dataStorage.json('categories.json')
|
||||
const categories = new Collection(categoriesContent).map(data => new Category(data))
|
||||
|
||||
const countriesContent = await storage.json('countries.json')
|
||||
const countriesContent = await dataStorage.json('countries.json')
|
||||
const countries = new Collection(countriesContent).map(data => new Country(data))
|
||||
|
||||
const languagesContent = await storage.json('languages.json')
|
||||
const languagesContent = await dataStorage.json('languages.json')
|
||||
const languages = new Collection(languagesContent).map(data => new Language(data))
|
||||
|
||||
const regionsContent = await storage.json('regions.json')
|
||||
const regionsContent = await dataStorage.json('regions.json')
|
||||
const regions = new Collection(regionsContent).map(data => new Region(data))
|
||||
|
||||
const subdivisionsContent = await storage.json('subdivisions.json')
|
||||
const subdivisionsContent = await dataStorage.json('subdivisions.json')
|
||||
const subdivisions = new Collection(subdivisionsContent).map(data => new Subdivision(data))
|
||||
|
||||
logger.info('loading streams...')
|
||||
const streams = await loadStreams({ channels, categories, languages })
|
||||
logger.info(`found ${streams.count()} streams`)
|
||||
|
||||
const generatorsLogger = new Logger({
|
||||
stream: await new Storage(LOGS_DIR).createStream(`generators.log`)
|
||||
|
@ -49,7 +45,6 @@ async function main() {
|
|||
|
||||
logger.info('generating categories/...')
|
||||
await new CategoriesGenerator({ categories, streams, logger: generatorsLogger }).generate()
|
||||
|
||||
logger.info('generating countries/...')
|
||||
await new CountriesGenerator({
|
||||
countries,
|
||||
|
@ -58,10 +53,8 @@ async function main() {
|
|||
subdivisions,
|
||||
logger: generatorsLogger
|
||||
}).generate()
|
||||
|
||||
logger.info('generating languages/...')
|
||||
await new LanguagesGenerator({ streams, logger: generatorsLogger }).generate()
|
||||
|
||||
logger.info('generating regions/...')
|
||||
await new RegionsGenerator({
|
||||
streams,
|
||||
|
@ -69,16 +62,12 @@ async function main() {
|
|||
subdivisions,
|
||||
logger: generatorsLogger
|
||||
}).generate()
|
||||
|
||||
logger.info('generating index.m3u...')
|
||||
await new IndexGenerator({ streams, logger: generatorsLogger }).generate()
|
||||
|
||||
logger.info('generating index.nsfw.m3u...')
|
||||
await new IndexNsfwGenerator({ streams, logger: generatorsLogger }).generate()
|
||||
|
||||
logger.info('generating index.category.m3u...')
|
||||
await new IndexCategoryGenerator({ streams, logger: generatorsLogger }).generate()
|
||||
|
||||
logger.info('generating index.country.m3u...')
|
||||
await new IndexCountryGenerator({
|
||||
streams,
|
||||
|
@ -87,10 +76,8 @@ async function main() {
|
|||
subdivisions,
|
||||
logger: generatorsLogger
|
||||
}).generate()
|
||||
|
||||
logger.info('generating index.language.m3u...')
|
||||
await new IndexLanguageGenerator({ streams, logger: generatorsLogger }).generate()
|
||||
|
||||
logger.info('generating index.region.m3u...')
|
||||
await new IndexRegionGenerator({ streams, regions, logger: generatorsLogger }).generate()
|
||||
}
|
||||
|
@ -110,11 +97,12 @@ async function loadStreams({
|
|||
const groupedCategories = categories.keyBy(category => category.id)
|
||||
const groupedLanguages = languages.keyBy(language => language.code)
|
||||
|
||||
const db = new Database(DB_DIR)
|
||||
const dbStreams = await db.load('streams.db')
|
||||
const docs = await dbStreams.find({})
|
||||
const streams = new Collection(docs as any[])
|
||||
.map((data: any) => new Stream(data))
|
||||
const storage = new Storage(STREAMS_DIR)
|
||||
const parser = new PlaylistParser({ storage })
|
||||
const files = await storage.list('**/*.m3u')
|
||||
let streams = await parser.parse(files)
|
||||
|
||||
streams = streams
|
||||
.orderBy([(stream: Stream) => stream.channel, (stream: Stream) => stream.url], ['asc', 'asc'])
|
||||
.uniqBy((stream: Stream) => stream.channel || _.uniqueId())
|
||||
.map((stream: Stream) => {
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
import { DB_DIR, DATA_DIR, STREAMS_DIR } from '../../constants'
|
||||
import { Database, Storage, Logger, Collection, Dictionary, IssueLoader } from '../../core'
|
||||
import { Stream, Playlist, Channel } from '../../models'
|
||||
import { DATA_DIR, STREAMS_DIR } from '../../constants'
|
||||
import { Storage, Logger, Collection, Dictionary, IssueLoader, PlaylistParser } from '../../core'
|
||||
import { Stream, Playlist, Channel, Issue } from '../../models'
|
||||
|
||||
let processedIssues = new Collection()
|
||||
let streams: Collection
|
||||
|
@ -10,19 +10,19 @@ async function main() {
|
|||
const logger = new Logger({ disabled: true })
|
||||
const loader = new IssueLoader()
|
||||
|
||||
logger.info('loading streams...')
|
||||
const db = new Database(DB_DIR)
|
||||
const docs = await db.load('streams.db')
|
||||
const dbStreams = await docs.find({})
|
||||
|
||||
streams = new Collection(dbStreams as any[]).map(data => new Stream(data))
|
||||
|
||||
const storage = new Storage(DATA_DIR)
|
||||
const channelsContent = await storage.json('channels.json')
|
||||
logger.info('loading channels from api...')
|
||||
const dataStorage = new Storage(DATA_DIR)
|
||||
const channelsContent = await dataStorage.json('channels.json')
|
||||
groupedChannels = new Collection(channelsContent)
|
||||
.map(data => new Channel(data))
|
||||
.keyBy((channel: Channel) => channel.id)
|
||||
|
||||
logger.info('loading streams...')
|
||||
const streamsStorage = new Storage(STREAMS_DIR)
|
||||
const parser = new PlaylistParser({ storage: streamsStorage })
|
||||
const files = await streamsStorage.list('**/*.m3u')
|
||||
streams = await parser.parse(files)
|
||||
|
||||
logger.info('removing broken streams...')
|
||||
await removeStreams(loader)
|
||||
|
||||
|
@ -32,25 +32,7 @@ async function main() {
|
|||
logger.info('add new streams...')
|
||||
await addStreams(loader)
|
||||
|
||||
logger.info('normalizing links...')
|
||||
streams = streams.map(stream => {
|
||||
stream.normalizeURL()
|
||||
return stream
|
||||
})
|
||||
|
||||
logger.info('sorting links...')
|
||||
streams = streams.orderBy(
|
||||
[
|
||||
(stream: Stream) => stream.name,
|
||||
(stream: Stream) => parseInt(stream.quality.replace('p', '')),
|
||||
(stream: Stream) => stream.label,
|
||||
(stream: Stream) => stream.url
|
||||
],
|
||||
['asc', 'desc', 'asc', 'asc']
|
||||
)
|
||||
|
||||
logger.info('saving...')
|
||||
const streamsStorage = new Storage(STREAMS_DIR)
|
||||
const groupedStreams = streams.groupBy((stream: Stream) => stream.filepath)
|
||||
for (let filepath of groupedStreams.keys()) {
|
||||
const streams = groupedStreams.get(filepath) || []
|
||||
|
@ -69,19 +51,22 @@ main()
|
|||
|
||||
async function removeStreams(loader: IssueLoader) {
|
||||
const issues = await loader.load({ labels: ['streams:remove', 'approved'] })
|
||||
issues.forEach((data: Dictionary) => {
|
||||
issues.forEach((issue: Issue) => {
|
||||
const data = issue.data
|
||||
if (data.missing('stream_url')) return
|
||||
|
||||
const removed = streams.remove((_stream: Stream) => _stream.url === data.get('stream_url'))
|
||||
if (removed.notEmpty()) {
|
||||
processedIssues.add(data.get('issue_number'))
|
||||
processedIssues.add(issue.number)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async function editStreams(loader: IssueLoader) {
|
||||
const issues = await loader.load({ labels: ['streams:edit', 'approved'] })
|
||||
issues.forEach((data: Dictionary) => {
|
||||
issues.forEach((issue: Issue) => {
|
||||
const data = issue.data
|
||||
|
||||
if (data.missing('stream_url')) return
|
||||
|
||||
let stream = streams.first(
|
||||
|
@ -111,13 +96,14 @@ async function editStreams(loader: IssueLoader) {
|
|||
streams.remove((_stream: Stream) => _stream.channel === stream.channel)
|
||||
streams.add(stream)
|
||||
|
||||
processedIssues.add(data.get('issue_number'))
|
||||
processedIssues.add(issue.number)
|
||||
})
|
||||
}
|
||||
|
||||
async function addStreams(loader: IssueLoader) {
|
||||
const issues = await loader.load({ labels: ['streams:add', 'approved'] })
|
||||
issues.forEach((data: Dictionary) => {
|
||||
issues.forEach((issue: Issue) => {
|
||||
const data = issue.data
|
||||
if (data.missing('channel_id') || data.missing('stream_url')) return
|
||||
if (streams.includes((_stream: Stream) => _stream.url === data.get('stream_url'))) return
|
||||
|
||||
|
@ -138,6 +124,6 @@ async function addStreams(loader: IssueLoader) {
|
|||
})
|
||||
|
||||
streams.add(stream)
|
||||
processedIssues.add(data.get('issue_number'))
|
||||
processedIssues.add(issue.number)
|
||||
})
|
||||
}
|
||||
|
|
|
@ -5,7 +5,6 @@ import chalk from 'chalk'
|
|||
import { transliterate } from 'transliteration'
|
||||
import _ from 'lodash'
|
||||
import { DATA_DIR, STREAMS_DIR } from '../../constants'
|
||||
import path from 'path'
|
||||
|
||||
program.argument('[filepath]', 'Path to file to validate').parse(process.argv)
|
||||
|
||||
|
@ -19,73 +18,70 @@ async function main() {
|
|||
const logger = new Logger()
|
||||
|
||||
logger.info(`loading blocklist...`)
|
||||
const storage = new Storage(DATA_DIR)
|
||||
const channelsContent = await storage.json('channels.json')
|
||||
const dataStorage = new Storage(DATA_DIR)
|
||||
const channelsContent = await dataStorage.json('channels.json')
|
||||
const channels = new Collection(channelsContent).map(data => new Channel(data))
|
||||
const blocklistContent = await storage.json('blocklist.json')
|
||||
const blocklistContent = await dataStorage.json('blocklist.json')
|
||||
const blocklist = new Collection(blocklistContent).map(data => new Blocked(data))
|
||||
|
||||
logger.info(`found ${blocklist.count()} records`)
|
||||
|
||||
let errors = new Collection()
|
||||
let warnings = new Collection()
|
||||
logger.info('loading streams...')
|
||||
const streamsStorage = new Storage(STREAMS_DIR)
|
||||
const parser = new PlaylistParser({ storage: streamsStorage })
|
||||
const files = program.args.length ? program.args : await streamsStorage.list('**/*.m3u')
|
||||
for (const filepath of files) {
|
||||
const file = new File(filepath)
|
||||
if (file.extension() !== 'm3u') continue
|
||||
const streams = await parser.parse(files)
|
||||
|
||||
logger.info(`found ${streams.count()} streams`)
|
||||
|
||||
let errors = new Collection()
|
||||
let warnings = new Collection()
|
||||
let groupedStreams = streams.groupBy((stream: Stream) => stream.filepath)
|
||||
for (const filepath of groupedStreams.keys()) {
|
||||
const streams = groupedStreams.get(filepath)
|
||||
if (!streams) continue
|
||||
|
||||
const file = new File(filepath)
|
||||
const [, countryCode] = file.basename().match(/([a-z]{2})(|_.*)\.m3u/i) || [null, '']
|
||||
|
||||
const log = new Collection()
|
||||
const buffer = new Dictionary()
|
||||
try {
|
||||
const relativeFilepath = filepath.replace(path.normalize(STREAMS_DIR), '')
|
||||
const playlist = await parser.parse(relativeFilepath)
|
||||
playlist.streams.forEach((stream: Stream) => {
|
||||
const channelNotInDatabase =
|
||||
stream.channel && !channels.first((channel: Channel) => channel.id === stream.channel)
|
||||
if (channelNotInDatabase) {
|
||||
log.add({
|
||||
type: 'warning',
|
||||
line: stream.line,
|
||||
message: `"${stream.channel}" is not in the database`
|
||||
})
|
||||
}
|
||||
streams.forEach((stream: Stream) => {
|
||||
const channelNotInDatabase =
|
||||
stream.channel && !channels.first((channel: Channel) => channel.id === stream.channel)
|
||||
if (channelNotInDatabase) {
|
||||
log.add({
|
||||
type: 'warning',
|
||||
line: stream.line,
|
||||
message: `"${stream.channel}" is not in the database`
|
||||
})
|
||||
}
|
||||
|
||||
const alreadyOnPlaylist = stream.url && buffer.has(stream.url)
|
||||
if (alreadyOnPlaylist) {
|
||||
log.add({
|
||||
type: 'warning',
|
||||
line: stream.line,
|
||||
message: `"${stream.url}" is already on the playlist`
|
||||
})
|
||||
} else {
|
||||
buffer.set(stream.url, true)
|
||||
}
|
||||
const alreadyOnPlaylist = stream.url && buffer.has(stream.url)
|
||||
if (alreadyOnPlaylist) {
|
||||
log.add({
|
||||
type: 'warning',
|
||||
line: stream.line,
|
||||
message: `"${stream.url}" is already on the playlist`
|
||||
})
|
||||
} else {
|
||||
buffer.set(stream.url, true)
|
||||
}
|
||||
|
||||
const channelId = generateChannelId(stream.name, countryCode)
|
||||
const blocked = blocklist.first(
|
||||
blocked =>
|
||||
stream.channel.toLowerCase() === blocked.channel.toLowerCase() ||
|
||||
channelId.toLowerCase() === blocked.channel.toLowerCase()
|
||||
)
|
||||
if (blocked) {
|
||||
log.add({
|
||||
type: 'error',
|
||||
line: stream.line,
|
||||
message: `"${stream.name}" is on the blocklist due to claims of copyright holders (${blocked.ref})`
|
||||
})
|
||||
}
|
||||
})
|
||||
} catch (error) {
|
||||
log.add({
|
||||
type: 'error',
|
||||
line: 0,
|
||||
message: error.message.toLowerCase()
|
||||
})
|
||||
}
|
||||
const channelId = generateChannelId(stream.name, countryCode)
|
||||
const blocked = blocklist.first(
|
||||
blocked =>
|
||||
stream.channel.toLowerCase() === blocked.channel.toLowerCase() ||
|
||||
channelId.toLowerCase() === blocked.channel.toLowerCase()
|
||||
)
|
||||
if (blocked) {
|
||||
log.add({
|
||||
type: 'error',
|
||||
line: stream.line,
|
||||
message: `"${stream.name}" is on the blocklist due to claims of copyright holders (${blocked.ref})`
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
if (log.notEmpty()) {
|
||||
logger.info(`\n${chalk.underline(filepath)}`)
|
||||
|
|
|
@ -1,36 +1,43 @@
|
|||
import { DATA_DIR } from '../../constants'
|
||||
import { Collection, Dictionary, IssueLoader, Storage } from '../../core'
|
||||
import { Blocked, Channel, Stream } from '../../models'
|
||||
import { DATA_DIR, STREAMS_DIR } from '../../constants'
|
||||
import { Collection, Dictionary, IssueLoader, Storage, Logger, PlaylistParser } from '../../core'
|
||||
import { Blocked, Channel, Issue, Stream } from '../../models'
|
||||
|
||||
async function main() {
|
||||
const logger = new Logger()
|
||||
const loader = new IssueLoader()
|
||||
|
||||
const storage = new Storage(DATA_DIR)
|
||||
|
||||
logger.info('loading channels from api...')
|
||||
const channelsContent = await storage.json('channels.json')
|
||||
const groupedChannels = new Collection(channelsContent)
|
||||
.map(data => new Channel(data))
|
||||
.groupBy((channel: Channel) => channel.id)
|
||||
|
||||
const streamsContent = await storage.json('streams.json')
|
||||
const groupedStreams = new Collection(streamsContent)
|
||||
.map(data => new Stream(data))
|
||||
.groupBy((stream: Stream) => stream.url)
|
||||
|
||||
logger.info('loading blocklist from api...')
|
||||
const blocklistContent = await storage.json('blocklist.json')
|
||||
const groupedBlocklist = new Collection(blocklistContent)
|
||||
.map(data => new Blocked(data))
|
||||
.groupBy((blocked: Blocked) => blocked.channel)
|
||||
|
||||
logger.info('loading streams...')
|
||||
const streamsStorage = new Storage(STREAMS_DIR)
|
||||
const parser = new PlaylistParser({ storage: streamsStorage })
|
||||
const files = await streamsStorage.list('**/*.m3u')
|
||||
const streams = await parser.parse(files)
|
||||
const groupedStreams = streams.groupBy((stream: Stream) => stream.url)
|
||||
|
||||
logger.info('loading issue from github...')
|
||||
const issues = await loader.load({ labels: ['streams:add'] })
|
||||
|
||||
logger.info('creating report...')
|
||||
const buffer = new Dictionary()
|
||||
const report = issues.map(data => {
|
||||
const channelId = data.get('channel_id') || undefined
|
||||
const streamUrl = data.get('stream_url') || undefined
|
||||
const report = issues.map((issue: Issue) => {
|
||||
const channelId = issue.data.get('channel_id') || undefined
|
||||
const streamUrl = issue.data.get('stream_url') || undefined
|
||||
|
||||
const result = new Dictionary({
|
||||
issueNumber: data.get('issue_number'),
|
||||
issueNumber: issue.number,
|
||||
channelId,
|
||||
status: undefined
|
||||
})
|
||||
|
|
|
@ -5,7 +5,6 @@ export const README_DIR = process.env.README_DIR || './.readme'
|
|||
export const API_DIR = process.env.API_DIR || './.api'
|
||||
export const DATA_DIR = process.env.DATA_DIR || './temp/data'
|
||||
export const LOGS_DIR = process.env.LOGS_DIR || './temp/logs'
|
||||
export const DB_DIR = process.env.DB_DIR || './temp/database'
|
||||
export const TESTING = process.env.NODE_ENV === 'test' ? true : false
|
||||
export const OWNER = 'iptv-org'
|
||||
export const REPO = 'iptv'
|
||||
|
|
|
@ -1,22 +0,0 @@
|
|||
import Datastore from '@seald-io/nedb'
|
||||
import * as path from 'path'
|
||||
|
||||
export class Database {
|
||||
rootDir: string
|
||||
|
||||
constructor(rootDir: string) {
|
||||
this.rootDir = rootDir
|
||||
}
|
||||
|
||||
async load(filepath: string) {
|
||||
const absFilepath = path.join(this.rootDir, filepath)
|
||||
|
||||
return new Datastore({
|
||||
filename: path.resolve(absFilepath),
|
||||
autoload: true,
|
||||
onload: (error: Error): any => {
|
||||
if (error) console.error(error.message)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
|
@ -1,4 +1,3 @@
|
|||
export * from './database'
|
||||
export * from './logger'
|
||||
export * from './playlistParser'
|
||||
export * from './numberParser'
|
||||
|
|
|
@ -1,33 +1,31 @@
|
|||
import { Dictionary } from './'
|
||||
import { Issue } from '../models'
|
||||
import _ from 'lodash'
|
||||
|
||||
const FIELDS = new Dictionary({
|
||||
'Channel ID': 'channel_id',
|
||||
'Channel ID (required)': 'channel_id',
|
||||
'Broken Link': 'stream_url',
|
||||
'Stream URL': 'stream_url',
|
||||
'Stream URL (optional)': 'stream_url',
|
||||
'Stream URL (required)': 'stream_url',
|
||||
Label: 'label',
|
||||
Quality: 'quality',
|
||||
'Channel Name': 'channel_name',
|
||||
'HTTP User-Agent': 'user_agent',
|
||||
'HTTP Referrer': 'http_referrer',
|
||||
Reason: 'reason',
|
||||
'What happened to the stream?': 'reason',
|
||||
'Possible Replacement (optional)': 'possible_replacement',
|
||||
Notes: 'notes',
|
||||
'Notes (optional)': 'notes'
|
||||
})
|
||||
|
||||
export class IssueParser {
|
||||
parse(issue: any): Dictionary {
|
||||
const data = new Dictionary()
|
||||
data.set('issue_number', issue.number)
|
||||
|
||||
const idDict = new Dictionary({
|
||||
'Channel ID': 'channel_id',
|
||||
'Channel ID (required)': 'channel_id',
|
||||
'Broken Link': 'stream_url',
|
||||
'Stream URL': 'stream_url',
|
||||
'Stream URL (optional)': 'stream_url',
|
||||
'Stream URL (required)': 'stream_url',
|
||||
Label: 'label',
|
||||
Quality: 'quality',
|
||||
'Channel Name': 'channel_name',
|
||||
'HTTP User-Agent': 'user_agent',
|
||||
'HTTP Referrer': 'http_referrer',
|
||||
Reason: 'reason',
|
||||
'What happened to the stream?': 'reason',
|
||||
'Possible Replacement (optional)': 'possible_replacement',
|
||||
Notes: 'notes',
|
||||
'Notes (optional)': 'notes'
|
||||
})
|
||||
|
||||
parse(issue: any): Issue {
|
||||
const fields = issue.body.split('###')
|
||||
|
||||
if (!fields.length) return data
|
||||
|
||||
const data = new Dictionary()
|
||||
fields.forEach((field: string) => {
|
||||
let [_label, , _value] = field.split(/\r?\n/)
|
||||
_label = _label ? _label.trim() : ''
|
||||
|
@ -35,7 +33,7 @@ export class IssueParser {
|
|||
|
||||
if (!_label || !_value) return data
|
||||
|
||||
const id: string = idDict.get(_label)
|
||||
const id: string = FIELDS.get(_label)
|
||||
const value: string = _value === '_No response_' || _value === 'None' ? '' : _value
|
||||
|
||||
if (!id) return
|
||||
|
@ -43,6 +41,6 @@ export class IssueParser {
|
|||
data.set(id, value)
|
||||
})
|
||||
|
||||
return data
|
||||
return new Issue({ number: issue.number, data })
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,6 +1,8 @@
|
|||
import parser from 'iptv-playlist-parser'
|
||||
import { Playlist, Stream } from '../models'
|
||||
import { Stream } from '../models'
|
||||
import { Collection, Storage } from './'
|
||||
import path from 'path'
|
||||
import { STREAMS_DIR } from '../constants'
|
||||
|
||||
export class PlaylistParser {
|
||||
storage: Storage
|
||||
|
@ -9,7 +11,19 @@ export class PlaylistParser {
|
|||
this.storage = storage
|
||||
}
|
||||
|
||||
async parse(filepath: string): Promise<Playlist> {
|
||||
async parse(files: string[]): Promise<Collection> {
|
||||
let streams = new Collection()
|
||||
|
||||
for (let filepath of files) {
|
||||
const relativeFilepath = filepath.replace(path.normalize(STREAMS_DIR), '')
|
||||
const _streams: Collection = await this.parseFile(relativeFilepath)
|
||||
streams = streams.concat(_streams)
|
||||
}
|
||||
|
||||
return streams
|
||||
}
|
||||
|
||||
async parseFile(filepath: string): Promise<Collection> {
|
||||
const streams = new Collection()
|
||||
|
||||
const content = await this.storage.read(filepath)
|
||||
|
@ -32,7 +46,7 @@ export class PlaylistParser {
|
|||
streams.add(stream)
|
||||
})
|
||||
|
||||
return new Playlist(streams)
|
||||
return streams
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -10,10 +10,12 @@ export class Storage {
|
|||
this.rootDir = path.normalize(rootDir || './')
|
||||
}
|
||||
|
||||
list(pattern: string): Promise<string[]> {
|
||||
return glob(pattern, {
|
||||
async list(pattern: string): Promise<string[]> {
|
||||
const files = await glob(pattern, {
|
||||
cwd: this.rootDir
|
||||
})
|
||||
|
||||
return files.sort()
|
||||
}
|
||||
|
||||
async createDir(dir: string): Promise<void> {
|
||||
|
|
|
@ -1,3 +1,4 @@
|
|||
export * from './issue'
|
||||
export * from './playlist'
|
||||
export * from './blocked'
|
||||
export * from './stream'
|
||||
|
|
16
scripts/models/issue.ts
Normal file
16
scripts/models/issue.ts
Normal file
|
@ -0,0 +1,16 @@
|
|||
import { Dictionary } from '../core'
|
||||
|
||||
type IssueProps = {
|
||||
number: number
|
||||
data: Dictionary
|
||||
}
|
||||
|
||||
export class Issue {
|
||||
number: number
|
||||
data: Dictionary
|
||||
|
||||
constructor({ number, data }: IssueProps) {
|
||||
this.number = number
|
||||
this.data = data
|
||||
}
|
||||
}
|
Loading…
Reference in a new issue