wip

2021-03-12 17:17:57 +03:00
parent 594197954e
commit b9be8e9d33
15 changed files with 1046 additions and 1540 deletions
--- a/bin/epg-grabber/index.js
+++ b/bin/epg-grabber/index.js
@@ -1,56 +1,32 @@
 #! /usr/bin/env node

-const fs = require('fs')
-const path = require('path')
-const axios = require('axios')
-const axiosCookieJarSupport = require('axios-cookiejar-support').default
-const tough = require('tough-cookie')
-const utils = require('./utils')
 const { Command } = require('commander')
 const program = new Command()
-const dayjs = require('dayjs')
-const utc = require('dayjs/plugin/utc')
-const timezone = require('dayjs/plugin/timezone')
-dayjs.extend(utc)
-dayjs.extend(timezone)
-dayjs.tz.setDefault('UTC')
-
-axiosCookieJarSupport(axios)
-
-const cookieJar = new tough.CookieJar()
+const utils = require('./utils')

 program
  .version('0.1.0', '-v, --version')
  .name('epg-grabber')
  .description('EPG grabber')
-  .usage('[options] [file-or-url]')
-  .option('-c, --config <config>', 'Path to [site].config.xml file', './')
-  .option('-s, --sites <sites>', 'Path to sites folder', './sites')
+  .option('-c, --config <config>', 'Path to [site].config.js file')
  .parse(process.argv)

 const options = program.opts()

-const config = utils.parseConfig(options.config)
-const sites = utils.loadSites(options.sites)
-
-const client = axios.create({
-  headers: { 'User-Agent': config.userAgent, Cookie: config.cookie },
-  withCredentials: true,
-  jar: cookieJar
-})
-
 async function main() {
  console.log('\r\nStarting...')
-  console.log(`Loading '${options.config}'...`)
-  const d = dayjs.utc()
+
+  const config = utils.loadConfig(options.config)
+  const client = utils.createHttpClient(config)
+  const channels = utils.parseChannels(config.channels)
+
+  const d = utils.getUTCDate()
  const dates = Array.from({ length: config.days }, (_, i) => d.add(i, 'd'))
-  const channels = config.channels
-  const requests = []
+  const queue = []
  channels.forEach(channel => {
-    const site = sites[channel.site]
    dates.forEach(date => {
-      requests.push({
-        url: site.url({ date, channel }),
+      queue.push({
+        url: config.url({ date, channel }),
        date,
        channel
      })
@@ -59,27 +35,17 @@ async function main() {

  console.log('Parsing:')
  let programs = []
-  for (let request of requests) {
+  for (let item of queue) {
    const progs = await client
-      .get(request.url)
+      .get(item.url)
      .then(response => {
-        const channel = request.channel
-        const site = sites[channel.site]
-        const date = request.date
-
-        const programs = site
-          .parser({
-            channel,
-            content: response.data,
-            date,
-            lang: config.lang || 'en'
-          })
-          .filter(p => p)
+        const parserOptions = Object.assign({}, item, config, { content: response.data })
+        const programs = config.parser(parserOptions).filter(i => i)

        console.log(
-          `  ${channel.site} - ${channel.xmltv_id} - ${date.format('MMM D, YYYY')} (${
-            programs.length
-          } programs)`
+          `  ${item.channel.site} - ${item.channel.xmltv_id} - ${item.date.format(
+            'MMM D, YYYY'
+          )} (${programs.length} programs)`
        )

        return programs
@@ -91,9 +57,11 @@ async function main() {
  }

  const xml = utils.convertToXMLTV({ config, channels, programs })
-  utils.createDir(path.dirname(config.filename))
-  utils.writeToFile(config.filename, xml)
-  console.log(`File '${config.filename}' successfully updated`)
+  const outputDir = utils.getDirectory(config.output)
+  utils.createDir(outputDir)
+  utils.writeToFile(config.output, xml)
+
+  console.log(`File '${config.output}' successfully saved`)
  console.log('Finish\r\n')
 }

--- a/bin/epg-grabber/utils.js
+++ b/bin/epg-grabber/utils.js
@@ -1,14 +1,66 @@
 const fs = require('fs')
 const path = require('path')
+const axios = require('axios')
+const axiosCookieJarSupport = require('axios-cookiejar-support').default
+const tough = require('tough-cookie')
 const convert = require('xml-js')
-const dayjs = require('dayjs')
 const glob = require('glob')
+const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
 const timezone = require('dayjs/plugin/timezone')
 dayjs.extend(utc)
 dayjs.extend(timezone)
+dayjs.tz.setDefault('UTC')
+axiosCookieJarSupport(axios)

 const utils = {}
+utils.loadConfig = function (file) {
+  if (!file) throw new Error('Path to [site].config.js is missing')
+  console.log(`Loading '${file}'...`)
+
+  const config = require(path.resolve(process.cwd(), file))
+
+  return Object.assign(
+    {},
+    {
+      userAgent:
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71',
+      days: 1,
+      cookie: '',
+      lang: 'en'
+    },
+    config
+  )
+}
+
+utils.parseChannels = function (file) {
+  if (!file) throw new Error('Path to [site].channels.xml is missing')
+  console.log(`Loading '${file}'...`)
+
+  const xml = fs.readFileSync(path.resolve(__dirname, file), {
+    encoding: 'utf-8'
+  })
+  const result = convert.xml2js(xml)
+  const site = result.elements.find(el => el.name === 'site')
+  const channels = site.elements.find(el => el.name === 'channels')
+
+  return channels.elements
+    .filter(el => el.name === 'channel')
+    .map(el => {
+      const channel = el.attributes
+      channel.name = el.elements.find(el => el.type === 'text').text
+      channel.site = channel.site || site.attributes.site
+
+      return channel
+    })
+}
+
+utils.sleep = function (ms) {
+  return function (x) {
+    return new Promise(resolve => setTimeout(() => resolve(x), ms))
+  }
+}
+
 utils.escapeString = function (string) {
  return string
    .toString()
@@ -23,31 +75,25 @@ utils.escapeString = function (string) {
 }

 utils.convertToXMLTV = function ({ config, channels, programs }) {
-  let output = '<?xml version="1.0" encoding="UTF-8" ?><tv>'
+  let output = `<?xml version="1.0" encoding="UTF-8" ?><tv>\r\n`

  for (let channel of channels) {
    const displayName = this.escapeString(channel.name)
-    output += `
-<channel id="${channel['xmltv_id']}"><display-name>${displayName}</display-name></channel>`
+    output += `<channel id="${channel['xmltv_id']}"><display-name>${displayName}</display-name></channel>\r\n`
  }

  for (let program of programs) {
    if (!program) continue

-    const start = program.start
-      ? dayjs.tz(program.start, config.timezone).format('YYYYMMDDHHmmss ZZ')
-      : ''
-    const stop = program.stop
-      ? dayjs.tz(program.stop, config.timezone).format('YYYYMMDDHHmmss ZZ')
-      : ''
+    const start = program.start ? dayjs(program.start).format('YYYYMMDDHHmmss ZZ') : ''
+    const stop = program.stop ? dayjs(program.stop).format('YYYYMMDDHHmmss ZZ') : ''
    const title = program.title ? this.escapeString(program.title) : ''
    const description = program.description ? this.escapeString(program.description) : ''
    const category = program.category ? this.escapeString(program.category) : ''
    const lang = program.lang ? program.lang : 'en'

    if (start && title) {
-      output += `
-<programme start="${start}"`
+      output += `<programme start="${start}"`

      if (stop) {
        output += ` stop="${stop}"`
@@ -63,67 +109,17 @@ utils.convertToXMLTV = function ({ config, channels, programs }) {
        output += `<category lang="${lang}">${category}</category>`
      }

-      output += '</programme>'
+      output += '</programme>\r\n'
    }
  }

-  output += '\r\n</tv>'
+  output += '</tv>'

  return output
 }

-utils.parseConfig = function (configPath) {
-  const xml = fs.readFileSync(path.resolve(process.cwd(), configPath), {
-    encoding: 'utf-8'
-  })
-  const result = convert.xml2js(xml)
-  const settings = result.elements.find(el => el.name === 'settings')
-  const filename = this.getElementText('filename', settings.elements)
-  const days = this.getElementText('days', settings.elements)
-  const userAgent = this.getElementText('user-agent', settings.elements)
-  const timezone = this.getElementText('timezone', settings.elements)
-  const cookie = this.getElementText('cookie', settings.elements)
-  const lang = this.getElementText('lang', settings.elements)
-  const channels = settings.elements
-    .filter(el => el.name === 'channel')
-    .map(el => {
-      const channel = el.attributes
-      channel.name = el.elements.find(el => el.type === 'text').text
-
-      return channel
-    })
-
-  return {
-    filename,
-    days,
-    userAgent,
-    timezone,
-    channels,
-    cookie,
-    lang
-  }
-}
-
-utils.getElementText = function (name, elements) {
-  const el = elements.find(el => el.name === name)
-
-  return el ? el.elements.find(el => el.type === 'text').text : null
-}
-
-utils.loadSites = function (sitesPath) {
-  const sites = {}
-  glob.sync(`${sitesPath}/*.js`).forEach(function (file) {
-    const name = path.parse(file).name
-    sites[name] = require(path.resolve(file))
-  })
-
-  return sites
-}
-
-utils.sleep = function (ms) {
-  return function (x) {
-    return new Promise(resolve => setTimeout(() => resolve(x), ms))
-  }
+utils.getDirectory = function (file) {
+  return path.dirname(file)
 }

 utils.createDir = function (dir) {
@@ -136,4 +132,19 @@ utils.writeToFile = function (filename, data) {
  fs.writeFileSync(path.resolve(__dirname, filename), data)
 }

+utils.createHttpClient = function (config) {
+  return axios.create({
+    headers: {
+      'User-Agent': config.userAgent,
+      Cookie: config.cookie
+    },
+    withCredentials: true,
+    jar: new tough.CookieJar()
+  })
+}
+
+utils.getUTCDate = function () {
+  return dayjs.utc()
+}
+
 module.exports = utils