diff --git a/.github/workflows/pushSiteMapToSearchEngine.yml b/.github/workflows/pushSiteMapToSearchEngine.yml new file mode 100644 index 0000000..a7b5e8d --- /dev/null +++ b/.github/workflows/pushSiteMapToSearchEngine.yml @@ -0,0 +1,36 @@ +name: Push Sitemap URLs to Search Engines + +on: + push: + schedule: + - cron: '0 23 * * *' # Runs at 23:00 UTC (7:00 AM Beijing Time) + workflow_dispatch: + inputs: + unconditional-invoking: + description: 'Push URLs unconditionally' + type: boolean + required: true + default: true + +jobs: + push-urls: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '18' + + - name: Install dependencies + run: npm install axios fast-xml-parser googleapis + + - name: Push Sitemap URLs to Search Engines + env: + URL: ${{ secrets.URL }} + BAIDU_TOKEN: ${{ secrets.BAIDU_TOKEN }} + BING_API_KEY: ${{ secrets.BING_API_KEY }} + GOOGLE_CREDENTIALS: ${{ secrets.GOOGLE_CREDENTIALS }} + run: node pushSiteMapToSearchEngine.js diff --git a/pushSiteMapToSearchEngine.js b/pushSiteMapToSearchEngine.js new file mode 100644 index 0000000..9cd8049 --- /dev/null +++ b/pushSiteMapToSearchEngine.js @@ -0,0 +1,159 @@ +const https = require('https') +const { XMLParser } = require('fast-xml-parser') +const axios = require('axios') +const { google } = require('googleapis') + +// Daily push quota, can be modified based on actual needs +const QUOTA = 100 + +async function parseSitemap(site) { + try { + const sitemapUrl = `${site}/sitemap.xml` + const response = await axios.get(sitemapUrl, { + httpsAgent: new https.Agent({ rejectUnauthorized: false }) + }) + + const parser = new XMLParser() + const result = parser.parse(response.data) + + // Handle different sitemap formats + let urls = [] + if (result.urlset && result.urlset.url) { + urls = Array.isArray(result.urlset.url) + ? result.urlset.url.map((u) => u.loc) + : [result.urlset.url.loc] + } + + return urls + } catch (error) { + console.error('Please check if your URL is correct.') + console.error( + 'The correct format should be a complete domain including https://, without sitemap.xml' + ) + console.error('Correct example: https://ghlcode.cn') + console.error('For details see: https://ghlcode.cn/fe032806-5362-4d82-b746-a0b26ce8b9d9') + console.error('Error details:', error.message) + return null + } +} + +async function pushToBing(site, urls, apiKey) { + const endpoint = `https://ssl.bing.com/webmaster/api.svc/json/SubmitUrlbatch?apikey=${apiKey}` + + const payload = { + siteUrl: site, + urlList: urls + } + + try { + const response = await axios.post(endpoint, payload, { + httpsAgent: new https.Agent({ rejectUnauthorized: false }) + }) + + if (response.status === 200) { + console.log('Successfully pushed to Bing.') + } + } catch (error) { + console.error('Error pushing to Bing:', error.response?.data?.Message || error.message) + } +} + +async function pushToBaidu(site, urls, token) { + const apiUrl = `http://data.zz.baidu.com/urls?site=${site}&token=${token}` + + try { + const response = await axios.post(apiUrl, urls.join('\\n'), { + headers: { 'Content-Type': 'text/plain' } + }) + + const result = response.data + if (result.success) { + console.log('Successfully pushed to Baidu.') + } else if (result.error) { + console.error('Error pushing to Baidu:', result.message) + } else { + console.error('Unknown response from Baidu:', result) + } + } catch (error) { + console.error('Error pushing to Baidu:', error.response?.data?.message || error.message) + } +} + +async function pushToGoogle(urls, credentials) { + try { + const auth = new google.auth.GoogleAuth({ + credentials, + scopes: ['https://www.googleapis.com/auth/indexing'] + }) + + const indexing = google.indexing({ version: 'v3', auth }) + + for (const url of urls) { + try { + await indexing.urlNotifications.publish({ + requestBody: { + url: url, + type: 'URL_UPDATED' + } + }) + console.log(`Successfully pushed ${url} to Google`) + } catch (error) { + console.error(`Error pushing ${url} to Google:`, error.message) + } + + // Rate limiting to avoid hitting Google's quotas + await new Promise((resolve) => setTimeout(resolve, 100)) + } + + console.log('Completed pushing to Google.') + } catch (error) { + console.error('Error setting up Google client:', error.message) + } +} + +async function main() { + // Get command line arguments + const args = process.argv.slice(2) + const url = process.env.URL || args[0] + const baiduToken = process.env.BAIDU_TOKEN + const bingApiKey = process.env.BING_API_KEY + const googleCredentials = process.env.GOOGLE_CREDENTIALS + ? JSON.parse(process.env.GOOGLE_CREDENTIALS) + : null + + if (!url) { + console.error('Please configure URL in Github Action Secrets') + console.error('For details see: https://ghlcode.cn/fe032806-5362-4d82-b746-a0b26ce8b9d9') + return + } + + // Parse URLs + const urls = await parseSitemap(url) + if (!urls || urls.length === 0) { + return + } + + // Check if URLs exceed quota + const selectedUrls = + urls.length > QUOTA ? urls.sort(() => Math.random() - 0.5).slice(0, QUOTA) : urls + + // Push to Bing + if (bingApiKey) { + console.log('Pushing to Bing, please wait...') + await pushToBing(url, selectedUrls, bingApiKey) + } + + // Push to Baidu + if (baiduToken) { + console.log('Pushing to Baidu, please wait...') + await pushToBaidu(url, selectedUrls, baiduToken) + } + + // Push to Google + if (googleCredentials) { + console.log('Pushing to Google, please wait...') + await pushToGoogle(selectedUrls, googleCredentials) + } +} + +main().catch(console.error)