-
Notifications
You must be signed in to change notification settings - Fork 9
/
resources.py
34 lines (30 loc) · 1023 Bytes
/
resources.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
# -*- coding: utf-8 -*-
from config import Config as con
import sys
import os
import helper
from helper import download_file
from helper import bcolors
from bs4 import BeautifulSoup
import json
import re
import requests
def download_course_data(conn, link, path):
page = con.session.get(helper.embbed_link(link))
soup = BeautifulSoup(page.text, 'html.parser')
dataset = soup.findAll('a', {
'href': re.compile('^https'),
'class': re.compile('^link-borderless')
})
titles = [x.text.strip() for x in dataset]
all_links = [x['href'] for x in dataset]
sys.stdout.write(
f'{bcolors.BOLD}Downloading dataset...{bcolors.ENDC}\n')
if not os.path.exists(path):
os.mkdir(path)
if(not os.path.exists(os.path.join(path, 'Dataset'))):
os.mkdir(os.path.join(path, 'Dataset'))
for link, title in zip(all_links, titles):
dir = os.path.join(path, 'Dataset', title)
dir = dir + '.' + link.split('.')[-1]
download_file(con, link, dir)