web-crawler/crawl.js at main · ShadowFax1731/web-crawler · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
const { JSDOM } = require('jsdom')


async function crawlPage(baseURL, currentURL, pages) {


    const base = new URL(baseURL)
    const current = new URL(currentURL)

    if (base.hostname !== current.hostname)
        return pages

    const normalizedCurrentURL = normalizeURL(currentURL)
    if (pages[normalizedCurrentURL] > 0) {
        pages[normalizedCurrentURL]++
        return pages
    }

    pages[normalizedCurrentURL] = 1

    console.log("Actively Crawling:", currentURL)

    try {

        const response = await fetch(currentURL)

        if (response.status > 399) {
            console.log("Error in fetch with status code:", response.status, "on page:", currentURL)
            return pages
        }

        const contentType = response.headers.get('content-type')

        if (!contentType.includes('text/html')) {
            console.log("Non HTML Response, Content Type:", contentType, "on page:", currentURL)
            return pages

        }

        const htmlBody = await response.text()

        const nextURLs = getURLsFromHTML(htmlBody, baseURL)

        for (const nextURL of nextURLs) {
            pages = await crawlPage(baseURL, nextURL, pages)

        }

    } catch (error) {
        console.log("Error in fetch:", error.message, "on page:", currentURL)
    }

    return pages

}

function getURLsFromHTML(htmlBody, baseURL) {
    const urls = []
    const dom = new JSDOM(htmlBody)
    const links = dom.window.document.querySelectorAll('a')

    for (const link of links)
        if (link.href.slice(0, 1) === '/') {
            //relative

            try {
                const url = new URL(`${baseURL}${link.href}`)
                urls.push(url.href)
            } catch (error) {
                console.log("Error with relative URL", error.message)
            }

        } else {
            //absolute URL
            try {
                const url = new URL(link.href)
                urls.push(url.href)
            } catch (error) {
                console.log("Error with absolute URL", error.message)
            }
        }

    return urls
}

function normalizeURL(urlString) {

    const url = new URL(urlString)

    const hostPath = `${url.hostname}${url.pathname}`

    if (hostPath.length > 0 && hostPath.slice(-1) === '/') {
        return hostPath.slice(0, -1)
    }

    return hostPath
}

module.exports = {
    normalizeURL,
    getURLsFromHTML,
    crawlPage
}