-
Notifications
You must be signed in to change notification settings - Fork 1
/
grabber.rb
144 lines (115 loc) · 3.57 KB
/
grabber.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
# frozen_string_literal: true
require 'faraday'
require 'json'
require 'down'
require 'fileutils'
require 'yaml'
DOWNLOAD_DIR = YAML.safe_load(File.read('config.yaml'))['download_dir']
def downloaded_ids
FileUtils.touch('downloaded_ids')
File.read('downloaded_ids').split("\n")
end
class Video
attr_accessor :url, :created_at, :id, :lecture
def initialize(url:, created_at:, id:, lecture:)
@created_at = created_at
@url = url
@id = id
@lecture = lecture
end
def download
dest = "#{lecture.dir}/#{filename}"
puts "Saving to #{dest}"
FileUtils.mkdir_p(lecture.dir)
Down.download(url, destination: dest)
puts 'Saved.'
File.write('downloaded_ids', id + "\n", mode: 'a')
end
def filename
"#{created_at}_#{lecture.course.gsub(' ', '-')}_#{lecture.code}_#{id}.mp4"
end
end
class Lecture
attr_accessor :username, :password, :base, :department, :year, :semester, :course, :code, :ldap
class << self
def from_json(filename)
json = JSON.parse(File.read(filename))
json.map do |lecture|
Lecture.new(
username: lecture['username'],
password: lecture['password'],
base: lecture['base'],
course: lecture['course'],
ldap: lecture['ldap']
)
end
end
end
def initialize(username:, password:, base:, course:, ldap:)
@username = username
@password = password
@base = base
@department, @year, @semester, @code = base.scan(%r{https://video\.ethz\.ch/lectures/(.*?)/(.*?)/(.*?)/(.*)}).first
@course = course
@ldap = ldap ? true : false
end
def missing_episodes
@missing_episodes ||= metadata['episodes'].reject { |e| downloaded_ids.include?(e['id']) }.map do |e|
id = e['id']
created_at = e['createdAt'].split('T')[0]
Video.new(id: id, url: best_video_url(id), created_at: created_at, lecture: self)
end
end
def dir
"#{DOWNLOAD_DIR}/#{department}/#{year}/#{semester}/#{course}"
end
private
def cookie
@cookie ||= (ldap? ? ldap_cookie : series_cookie)
end
def series_cookie
Faraday.post(base + '.series-login.json') do |req|
req.params['username'] = username
req.params['password'] = password
end.headers['set-cookie']
end
def ldap_cookie
Faraday.post(ldap_url) do |req|
req.params['j_username'] = username
req.params['j_password'] = password
req.params['j_validate'] = true
req.params['_charset_'] = 'utf-8'
end.headers['set-cookie']
end
def ldap_url
"https://video.ethz.ch/lectures/#{department}/#{year}/#{semester}/j_security_check"
end
def metadata
@metadata ||= JSON.parse(Faraday.get(base + '.series-metadata.json').body).to_h
end
def episode_videos(id)
JSON.parse(Faraday.get('https://video.ethz.ch/.episode-video.json') do |req|
req.params['recordId'] = id
req.headers['cookie'] = cookie if protected?
end.body).to_h
end
def best_video_url(id)
episode_videos(id)['streams'][0]['sources']['mp4'].max { |a, b| a['res']['w'].to_i - b['res']['w'].to_i }['src']
end
def protected?
username && password
end
def ldap?
ldap
end
end
def download_missing_lectures
lectures = Lecture.from_json 'lectures.json'
lectures.reject! { |lecture| lecture.missing_episodes.empty? }
lectures.each do |lecture|
puts "Found #{lecture.missing_episodes.count} new #{lecture.missing_episodes.count > 1 ? 'episodes' : 'episode'} for lecture \"#{lecture.course}\"!"
lecture.missing_episodes.each(&:download)
end
puts 'No new episodes found.' if lectures.empty?
end
download_missing_lectures