lib/parser.js - master - streamlive/embedstream

lib/parser.js
338 строк10 KB
TwixoffStudio
Create: package.json, .gitkeep, parser.js, store.js, embed.ejs, error.ejs, index.ejs, .env.example, .gitignore, LICENSE, README.md, server.js
10 авг 2026, 11:06
Верифицирован
10 авг 2026, 11:066adfd59
/**
 * EmbedStream URL parser
 * Extracts Open Graph / oEmbed / platform-specific metadata
 * Proper video covers (og:image), titles, descriptions, tags.
 */

const cheerio = require('cheerio');
const { URL } = require('url');

const UA =
  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';

const TIMEOUT = 12000;

async function fetchHtml(url) {
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), TIMEOUT);
  try {
    const res = await fetch(url, {
      signal: controller.signal,
      headers: {
        'User-Agent': UA,
        Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
        'Accept-Language': 'ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7',
        'Cache-Control': 'no-cache',
      },
      redirect: 'follow',
    });
    clearTimeout(timer);
    if (!res.ok) throw new Error(`HTTP ${res.status}`);
    const html = await res.text();
    return { html, finalUrl: res.url };
  } catch (e) {
    clearTimeout(timer);
    throw e;
  }
}

function extractOg($, baseUrl) {
  const get = (prop) => {
    const el =
      $(`meta[property="${prop}"]`).attr('content') ||
      $(`meta[name="${prop}"]`).attr('content') ||
      $(`meta[property="og:${prop}"]`).attr('content') ||
      $(`meta[name="twitter:${prop}"]`).attr('content');
    return el ? el.trim() : null;
  };

  const title =
    get('og:title') ||
    get('twitter:title') ||
    $('title').first().text().trim() ||
    null;

  const description =
    get('og:description') ||
    get('twitter:description') ||
    get('description') ||
    null;

  let image =
    get('og:image') ||
    get('og:image:secure_url') ||
    get('twitter:image') ||
    get('twitter:image:src') ||
    null;

  if (image && !image.startsWith('http')) {
    try {
      image = new URL(image, baseUrl).href;
    } catch {}
  }

  const video =
    get('og:video') ||
    get('og:video:url') ||
    get('og:video:secure_url') ||
    get('twitter:player') ||
    null;

  const siteName = get('og:site_name') || null;
  const type = get('og:type') || 'website';

  // tags / keywords
  let tags = [];
  const keywords = get('keywords') || get('news_keywords');
  if (keywords) {
    tags = keywords
      .split(/[,;]/)
      .map((t) => t.trim())
      .filter(Boolean)
      .slice(0, 15);
  }
  // also from article:tag
  $('meta[property="article:tag"]').each((_, el) => {
    const t = $(el).attr('content');
    if (t) tags.push(t.trim());
  });
  tags = [...new Set(tags)].slice(0, 15);

  return {
    title,
    description,
    image,
    video,
    siteName,
    type,
    tags,
  };
}

/** Platform-specific enrichment */
function detectPlatform(url) {
  try {
    const u = new URL(url);
    const host = u.hostname.replace(/^www\./, '').toLowerCase();

    if (host.includes('youtube.com') || host === 'youtu.be') return 'youtube';
    if (host.includes('youtu.be')) return 'youtube';
    if (host.includes('vk.com') || host.includes('vkvideo.ru') || host.includes('vk.ru')) return 'vk';
    if (host.includes('ok.ru') || host.includes('odnoklassniki.ru')) return 'ok';
    if (host.includes('rutube.ru')) return 'rutube';
    if (host.includes('tiktok.com')) return 'tiktok';
    if (host.includes('instagram.com')) return 'instagram';
    if (host.includes('vimeo.com')) return 'vimeo';
    if (host.includes('twitch.tv')) return 'twitch';
    if (host.includes('dropbox.com')) return 'dropbox';
    if (host.includes('drive.google.com') || host.includes('docs.google.com')) return 'gdrive';
    if (host.includes('disk.yandex.') || host.includes('yadi.sk')) return 'yandex';
    if (host.includes('dailymotion.com')) return 'dailymotion';
    if (host.includes('soundcloud.com')) return 'soundcloud';
    if (host.includes('t.me') || host.includes('telegram.org')) return 'telegram';
    if (/\.(mp4|webm|m3u8|mpd)(\?|$)/i.test(u.pathname + u.search)) return 'direct';
    return 'generic';
  } catch {
    return 'generic';
  }
}

function youtubeId(url) {
  try {
    const u = new URL(url);
    if (u.hostname === 'youtu.be') return u.pathname.slice(1).split('/')[0];
    if (u.searchParams.get('v')) return u.searchParams.get('v');
    const m = u.pathname.match(/\/(embed|v|shorts|live)\/([^/?]+)/);
    return m ? m[2] : null;
  } catch {
    return null;
  }
}

async function tryOembed(url, platform) {
  const endpoints = {
    youtube: `https://www.youtube.com/oembed?url=${encodeURIComponent(url)}&format=json`,
    vimeo: `https://vimeo.com/api/oembed.json?url=${encodeURIComponent(url)}`,
    soundcloud: `https://soundcloud.com/oembed?url=${encodeURIComponent(url)}&format=json`,
    dailymotion: `https://www.dailymotion.com/services/oembed?url=${encodeURIComponent(url)}`,
    // TikTok oEmbed often blocked, skip
  };
  const ep = endpoints[platform];
  if (!ep) return null;
  try {
    const controller = new AbortController();
    const t = setTimeout(() => controller.abort(), 8000);
    const res = await fetch(ep, {
      signal: controller.signal,
      headers: { 'User-Agent': UA },
    });
    clearTimeout(t);
    if (!res.ok) return null;
    return await res.json();
  } catch {
    return null;
  }
}

function buildEmbedUrl(platform, originalUrl, meta) {
  switch (platform) {
    case 'youtube': {
      const id = youtubeId(originalUrl);
      return id ? `https://www.youtube.com/embed/${id}?rel=0&modestbranding=1` : null;
    }
    case 'vimeo': {
      try {
        const m = originalUrl.match(/vimeo\.com\/(?:video\/)?(\d+)/);
        return m ? `https://player.vimeo.com/video/${m[1]}` : null;
      } catch {
        return null;
      }
    }
    case 'rutube': {
      try {
        const m = originalUrl.match(/rutube\.ru\/video\/([a-f0-9]+)/i);
        return m ? `https://rutube.ru/play/embed/${m[1]}` : null;
      } catch {
        return null;
      }
    }
    case 'dailymotion': {
      try {
        const m = originalUrl.match(/dailymotion\.com\/video\/([a-zA-Z0-9]+)/);
        return m ? `https://www.dailymotion.com/embed/video/${m[1]}` : null;
      } catch {
        return null;
      }
    }
    case 'twitch': {
      // channel or video
      try {
        const u = new URL(originalUrl);
        if (u.pathname.includes('/videos/')) {
          const id = u.pathname.split('/videos/')[1];
          return `https://player.twitch.tv/?video=${id}&parent=localhost`;
        }
        const channel = u.pathname.replace(/^\//, '').split('/')[0];
        if (channel) return `https://player.twitch.tv/?channel=${channel}&parent=localhost`;
      } catch {}
      return null;
    }
    case 'direct':
      return originalUrl;
    default:
      return meta.video || null;
  }
}

/**
 * Main entry
 * @param {string} inputUrl
 * @returns {Promise<object>}
 */
async function parseUrl(inputUrl) {
  let url = inputUrl.trim();
  if (!/^https?:\/\//i.test(url)) url = 'https://' + url;

  let finalUrl = url;
  let platform = detectPlatform(url);
  let meta = {
    title: null,
    description: null,
    image: null,
    video: null,
    siteName: null,
    type: 'website',
    tags: [],
  };
  let embedUrl = null;
  let htmlSource = null;

  // 1. Try oEmbed first for known platforms
  if (['youtube', 'vimeo', 'soundcloud', 'dailymotion'].includes(platform)) {
    const oembed = await tryOembed(url, platform);
    if (oembed) {
      meta.title = oembed.title || meta.title;
      meta.description = oembed.description || oembed.author_name || meta.description;
      meta.image = oembed.thumbnail_url || meta.image;
      meta.siteName = oembed.provider_name || meta.siteName;
      if (oembed.html) {
        // extract src from iframe if present
        const m = oembed.html.match(/src=["']([^"']+)["']/);
        if (m) embedUrl = m[1];
      }
    }
  }

  // 2. Always try HTML OG scrape (even after oEmbed for better tags/desc)
  try {
    const { html, finalUrl: f } = await fetchHtml(url);
    finalUrl = f;
    htmlSource = html;
    platform = detectPlatform(finalUrl) || platform;
    const $ = cheerio.load(html);
    const og = extractOg($, finalUrl);

    meta.title = meta.title || og.title;
    meta.description = meta.description || og.description;
    meta.image = meta.image || og.image;
    meta.video = meta.video || og.video;
    meta.siteName = meta.siteName || og.siteName;
    meta.type = og.type || meta.type;
    if (og.tags.length) meta.tags = [...new Set([...meta.tags, ...og.tags])].slice(0, 15);
  } catch (err) {
    // continue with what we have (oEmbed may have saved us)
    if (!meta.title && !meta.image) {
      throw new Error(`Не удалось получить данные: ${err.message}`);
    }
  }

  // 3. Build clean embed player URL
  if (!embedUrl) {
    embedUrl = buildEmbedUrl(platform, finalUrl, meta);
  }

  // 4. Fallbacks
  if (!meta.title) {
    try {
      const u = new URL(finalUrl);
      meta.title = u.hostname + u.pathname;
    } catch {
      meta.title = 'Без названия';
    }
  }

  // Clean title a bit
  if (meta.title) {
    meta.title = meta.title
      .replace(/\s*[|\-–—]\s*(YouTube|VK|ВКонтакте|Rutube|TikTok|Instagram|Vimeo).*$/i, '')
      .trim()
      .slice(0, 200);
  }

  if (meta.description) {
    meta.description = meta.description.slice(0, 500);
  }

  // Restricted platforms (RF law note)
  const restricted = ['instagram', 'facebook', 'x.com', 'twitter.com'].some((d) =>
    finalUrl.toLowerCase().includes(d)
  );

  return {
    url: finalUrl,
    originalUrl: inputUrl,
    platform,
    title: meta.title,
    description: meta.description,
    image: meta.image,
    video: meta.video,
    embedUrl,
    siteName: meta.siteName,
    type: meta.type,
    tags: meta.tags,
    restricted,
    fetchedAt: new Date().toISOString(),
  };
}

module.exports = { parseUrl, detectPlatform };