IMDB_Scraper_API/server.js at master · migtam628/IMDB_Scraper_API · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
var express = require('express');
var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');
var app = express();

app.get('/scrape', function(req, res) {
    var id = req.query.id
    var title = req.query.title
    var url = `https://www.imdb.com/title/${id}`;
    // url1 = `https://www.imdb.com/find?ref_=nv_sr_fn&q=${encodeURI(title)}&s=all`;
    request(url, function(error, response, html) {
        if (!error) {
            var $ = cheerio.load(html);

            var json = {
                title: "",
                release_date: "",
                rating: "",
                runtime: "",
                keywords: "",
                country: "",
                language: "",
                cast: [],
                plot: "",
                storyline: "",
                filming_locations: "",
                poster: "",
                backdrop: ""
            };

            $('.title_wrapper').filter(function() {
                var data = $(this);
                title = data.children().first().text().trim();
                release = data.children().last().children().last().text().trim();
                json.title = title;
                json.release_date = release;
            })

            $('.ratingValue').filter(function() {
                var data = $(this);
                rating = data.text().trim();
                json.rating = rating;
            })

            $('time').filter(function() {
                var data = $(this);
                runtime = data.text().trim();
                json.runtime = runtime;
            })

            $('.slate').filter(function() {
                var data = $(this);
                backdrop = data.children().first().children().first().attr('src')
                json.backdrop = backdrop;
            })

            $('.poster').filter(function() {
                var data = $(this);
                poster = data.children().first().children().first().attr('src')
                json.poster = poster;
            })

            $('.summary_text').filter(function() {
                var data = $(this);
                plot = data.text().trim();
                json.plot = plot;
            })

            $('.cast_list').filter(function() {
                var data = $(this);
                cast = data.children().first().text().trim();
                newCast = cast.replace(/[\n]+/g, "").trim()
                newCast1 = newCast.replace('Cast overview, first billed only: ', "").trim()
                json.cast = newCast1.replace(/['                                ...                              ']+/g, " ").trim();
            })

            $('.article#titleStoryLine').filter(() => {
                var data = $(this);
                storyline = $('.inline.canwrap').children('p').children('span').text().trim()
                keywords = $('.see-more.inline.canwrap').children('a').text()
                json.storyline = storyline.trim()
                json.keywords = keywords.trim().replace(/[" "]+/g, ", ")
            })

            $('.article#titleDetails').filter(() => {
                var data = $(this);
                lo = $('div.txt-block').eq(8).text();
                loc = lo.replace(/[\n]+/g, "");
                loca = loc.replace('Filming Locations:', "");
                locat = loca.replace("»", "");
                json.filming_locations = locat.replace("See more", "").trim();

                l = $('div.txt-block').eq(5).text();
                la = l.replace(/[\n]+/g, "");
                lan = la.replace('Language:', "");
                lang = lan.replace(/['            |        ']+/g, ", ").trim();
                langu = lang.replace('|', "")
                json.language = langu.substring(0, langu.length - 1).trim();

                c = $('div.txt-block').eq(4).text();
                co = c.replace(/[\n]+/g, "");
                cou = co.replace('Country:', "");
                coun = cou.replace("»", "");
                json.country = coun.replace("See more", "").trim();

            })

        }

        fs.writeFile(`${id}.json`, JSON.stringify(json, null, 4), function(err) {
            // fs.writeFile(`/Users/migtamrod/Desktop/${id}.json`, JSON.stringify(json, null, 4), function(err) {
            if (!err) {
                console.log(`File successfully written! - Check your project directory for the ${id}.json file`);
            } else {
                console.log({ err })
            }
        })

        res.json(JSON.parse(JSON.stringify(json)))
    })
})

app.listen('8081')
console.log('http://localhost:8081');
exports = module.exports = app;