Add engagement rating, rating, likes, dislikes, and form of logging (primitive but in the hope of reverse-engineering requests)

This commit is contained in:
Omar Roth 2017-11-28 19:59:51 -06:00
parent f9f20f4af0
commit 7feec0c00d
6 changed files with 49 additions and 76 deletions

View File

@ -6,7 +6,7 @@ shards:
kemal:
github: kemalcr/kemal
version: 0.21.0
commit: 8cb9770ec3c6cf5897e644229dad8d0b5c360941
kilt:
github: jeromegn/kilt

View File

@ -11,6 +11,7 @@ targets:
dependencies:
kemal:
github: kemalcr/kemal
branch: master
pg:
github: will/crystal-pg

View File

@ -3,93 +3,63 @@ require "json"
require "kemal"
require "pg"
require "xml"
require "./url_encoded"
macro templated(filename)
render "views/#{{{filename}}}.ecr", "views/layout.ecr"
end
# pg = DB.open("postgres://kemal@visor/dev")
alias Type = String | Hash(String, Type)
def object_to_hash(value)
object = {} of String => Type
items = value.split("&")
items.each do |item|
key, value = item.split("=")
value = URI.unescape(value)
object[key] = parse_uri(value)
end
return object
end
def array_to_hash(value)
array = {} of String => Type
items = value.split(",")
count = 0
items.each do |item|
array[count.to_s] = parse_uri(item)
count += 1
end
return array
end
def parse_uri(value)
if value.starts_with?("http") || value.starts_with?("[")
return value
else
if value.includes?(",")
return array_to_hash(value)
elsif value.includes?("&")
return object_to_hash(value)
else
return value
end
end
end
context = OpenSSL::SSL::Context::Client.insecure
client = HTTP::Client.new("www.youtube.com", 443, context)
fmt_file = File.open("temp/fmt_stream")
get "/" do |env|
templated "index"
end
get "/watch/:video_id" do |env|
video_id = env.params.url["video_id"]
video_info_encoded = HTTP::Client.get("https://www.youtube.com/get_video_info?video_id=#{video_id}&el=info&ps=default&eurl=&gl=US&hl=en", nil, nil, tls = context).body
video_info = object_to_hash(video_info_encoded)
body = client.get("/watch?v=#{video_id}").body
doc = XML.parse(body)
client = HTTP::Client.new("www.youtube.com", 443, context)
video_info = client.get("/get_video_info?video_id=#{video_id}&el=info&ps=default&eurl=&gl=US&hl=en").body
video_info = HTTP::Params.parse(video_info)
pageContent = client.get("/watch?v=#{video_id}").body
doc = XML.parse(pageContent)
fmt_stream = [] of HTTP::Params
video_info["url_encoded_fmt_stream_map"].split(",") do |string|
fmt_stream << HTTP::Params.parse(string)
end
File.write("temp/#{video_id}", video_info)
File.write("temp/#{video_id}_manifest", video_info["dashmpd"])
File.open("temp/#{video_id}_fmt_stream_0", "a+").puts fmt_stream[0]["url"]
File.open("temp/#{video_id}_fmt_stream_1", "a+").puts fmt_stream[1]["url"]
File.open("temp/#{video_id}_fmt_stream_2", "a+").puts fmt_stream[2]["url"]
File.open("temp/#{video_id}_fmt_stream_3", "a+").puts fmt_stream[3]["url"]
fmt_stream.reverse! # We want lowest quality first
# css query [title="I like this"] > span
likes = doc.xpath_node(%q(//button[@title="I like this"]/span))
if likes
likes = likes.content
likes = likes.content.delete(",").to_i
else
likes = "n/a"
likes = 1
end
# css query [title="I dislike this"] > span
dislikes = doc.xpath_node(%q(//button[@title="I dislike this"]/span))
if dislikes
dislikes.content
dislikes = dislikes.content.delete(",").to_i
else
dislikes = "n/a"
dislikes = 1
end
File.write("video_info/#{video_id}", video_info.to_json)
engagement = ((dislikes.to_f32 + likes.to_f32)*100 / video_info["view_count"].to_i).to_i
calculated_rating = likes.to_f32/(likes.to_f32 + dislikes.to_f32)*4 + 1
templated "watch"
end
# get "/listen/:video_id" do |env|
# video_id = env.params.url["video_id"]
# video_info_encoded = HTTP::Client.get("https://www.youtube.com/get_video_info?video_id=#{video_id}&el=info&ps=default&eurl=&gl=US&hl=en", nil, nil, tls = context).body
# video_info = object_to_hash(video_info_encoded)
# File.write("video_info/#{video_id}", video_info.to_json)
# templated "listen"
# end
public_folder "assets"
Kemal.run

View File

@ -1,6 +1,5 @@
<% title = URI.unescape(video_info["title"].as(String), true) %>
<h1><%= title %></h1>
<video style="width: 100%" poster="<%= video_info["iurlhq720"] %>" controls>
<h1><%= video_info["title"] %></h1>
<video style="width: 100%" poster="<%= video_info.has_key?("iurlhq720") ? video_info["iurlhq720"] : video_info["iurl"] %>" controls>
<% video_info["adaptive_fmts"].as(Hash).each do |key, value| %>
<% url = value["url"] %>
<% type = value["type"].to_s.split(";")[0] %>

View File

@ -1,20 +1,23 @@
<% title = URI.unescape(video_info["title"].as(String), true) %>
<h1><%= title %></h1>
<video style="width: 100%" poster="<%= video_info["iurl"] %>" controls>
<% video_info["url_encoded_fmt_stream_map"].as(Hash).each do |key, value| %>
<% url = value["url"] %>
<% type = value["type"]["0"].to_s.split(";")[0] %>
<source src="<%= url %>" type="<%= type %>">
<video style="width: 100%" poster="<%= video_info.has_key?("iurlhq720") ? video_info["iurlhq720"] : video_info["iurlmq"] %>" controls>
<% fmt_stream.each do |fmt| %>
<source src="<%= fmt["url"] %>" type="<%= fmt["type"].split(";")[0] %>">
<% end %>
</video>
<h1><%= video_info["title"] %></h1>
<div class="pure-g">
<div class="pure-u-1 pure-u-md-1-5">
<p>Likes: <%= likes %></p>
<p>Dislikes: <%= dislikes %></p>
<p>+ <%= likes %></p>
<p>-&nbsp; <%= dislikes %></p>
</div>
<div class="pure-u-1 pure-u-md-3-5"></div>
<div class="pure-u-1 pure-u-md-1-5">
<div class="pure-u-1 pure-u-md-3-5">
<p>Views : <%= video_info["view_count"] %></p>
<p>Rating : <%= video_info["avg_rating"] %></p>
<p>Calculated Rating : <%= calculated_rating %></p>
<p>Engagement : <%= engagement %>%</p>
</div>
<div class="pure-u-1 pure-u-md-1-5">
<% fmt_stream.each do |fmt| %>
<p><%= fmt["quality"] %></p>
<% end %>
</div>
</div>

View File

@ -38,7 +38,7 @@ https://www.youtube.com/embed/:ID
https://www.youtube.com/get_video_info?video_id= + video_id +
&el=info&ps=default&eurl=&gl=US&hl=en"&el=info&ps=default&eurl=&gl=US&hl=en
&el=info&ps=default&eurl=&gl=US&hl=en
```