From 1156b0998758ee803c7e8ae0cc2beb5181c232a3 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 21 Jun 2019 21:41:41 -0700 Subject: Refactor search page --- youtube/search.py | 119 ++++++++++++++++++++++++------------------------------ 1 file changed, 52 insertions(+), 67 deletions(-) (limited to 'youtube/search.py') diff --git a/youtube/search.py b/youtube/search.py index 0cef0f3..fcc352f 100644 --- a/youtube/search.py +++ b/youtube/search.py @@ -1,16 +1,12 @@ -from youtube import util, html_common, yt_data_extract, proto +from youtube import util, yt_data_extract, proto, local_playlist +from youtube import yt_app import json import urllib -import html -from string import Template import base64 from math import ceil - - -with open("yt_search_results_template.html", "r") as file: - yt_search_results_template = file.read() - +from flask import request +import flask # Sort: 1 # Upload date: 2 @@ -58,41 +54,32 @@ def get_search_json(query, page, autocorrect, sort, filters): content = util.fetch_url(url, headers=headers, report_text="Got search results") info = json.loads(content) return info - -showing_results_for = Template(''' -
Showing results for $corrected_query
-
Search instead for $original_query
-''') -did_you_mean = Template(''' -
Did you mean $corrected_query
-''') -def get_search_page(env, start_response): - start_response('200 OK', [('Content-type','text/html'),]) - parameters = env['parameters'] - if len(parameters) == 0: - return html_common.yt_basic_template.substitute( - page_title = "Search", - header = html_common.get_header(), - style = '', - page = '', - ).encode('utf-8') - query = parameters["query"][0] - page = parameters.get("page", "1")[0] - autocorrect = int(parameters.get("autocorrect", "1")[0]) - sort = int(parameters.get("sort", "0")[0]) + +@yt_app.route('/search') +def get_search_page(): + if len(request.args) == 0: + return flask.render_template('base.html', title="Search") + + if 'query' not in request.args: + abort(400) + + query = request.args.get("query") + page = request.args.get("page", "1") + autocorrect = int(request.args.get("autocorrect", "1")) + sort = int(request.args.get("sort", "0")) filters = {} - filters['time'] = int(parameters.get("time", "0")[0]) - filters['type'] = int(parameters.get("type", "0")[0]) - filters['duration'] = int(parameters.get("duration", "0")[0]) + filters['time'] = int(request.args.get("time", "0")) + filters['type'] = int(request.args.get("type", "0")) + filters['duration'] = int(request.args.get("duration", "0")) info = get_search_json(query, page, autocorrect, sort, filters) estimated_results = int(info[1]['response']['estimatedResults']) estimated_pages = ceil(estimated_results/20) results = info[1]['response']['contents']['twoColumnSearchResultsRenderer']['primaryContents']['sectionListRenderer']['contents'][0]['itemSectionRenderer']['contents'] - - corrections = '' - result_list_html = "" + + parsed_results = [] + corrections = {'type': None} for renderer in results: type = list(renderer.keys())[0] if type == 'shelfRenderer': @@ -102,41 +89,39 @@ def get_search_page(env, start_response): corrected_query_string = parameters.copy() corrected_query_string['query'] = [renderer['correctedQueryEndpoint']['searchEndpoint']['query']] corrected_query_url = util.URL_ORIGIN + '/search?' + urllib.parse.urlencode(corrected_query_string, doseq=True) - corrections = did_you_mean.substitute( - corrected_query_url = corrected_query_url, - corrected_query = yt_data_extract.format_text_runs(renderer['correctedQuery']['runs']), - ) + + corrections = { + 'type': 'did_you_mean', + 'corrected_query': yt_data_extract.format_text_runs(renderer['correctedQuery']['runs']), + 'corrected_query_url': corrected_query_url, + } continue if type == 'showingResultsForRenderer': renderer = renderer[type] no_autocorrect_query_string = parameters.copy() no_autocorrect_query_string['autocorrect'] = ['0'] no_autocorrect_query_url = util.URL_ORIGIN + '/search?' + urllib.parse.urlencode(no_autocorrect_query_string, doseq=True) - corrections = showing_results_for.substitute( - corrected_query = yt_data_extract.format_text_runs(renderer['correctedQuery']['runs']), - original_query_url = no_autocorrect_query_url, - original_query = html.escape(renderer['originalQuery']['simpleText']), - ) + + corrections = { + 'type': 'showing_results_for', + 'corrected_query': yt_data_extract.format_text_runs(renderer['correctedQuery']['runs']), + 'original_query_url': no_autocorrect_query_url, + 'original_query': renderer['originalQuery']['simpleText'], + } continue - result_list_html += html_common.renderer_html(renderer, current_query_string=env['QUERY_STRING']) - - page = int(page) - if page <= 5: - page_start = 1 - page_end = min(9, estimated_pages) - else: - page_start = page - 4 - page_end = min(page + 4, estimated_pages) - - - result = Template(yt_search_results_template).substitute( - header = html_common.get_header(query), - results = result_list_html, - page_title = query + " - Search", - search_box_value = html.escape(query), - number_of_results = '{:,}'.format(estimated_results), - number_of_pages = '{:,}'.format(estimated_pages), - page_buttons = html_common.page_buttons_html(page, estimated_pages, util.URL_ORIGIN + "/search", env['QUERY_STRING']), - corrections = corrections - ) - return result.encode('utf-8') + + info = yt_data_extract.parse_info_prepare_for_html(renderer) + if info['type'] != 'unsupported': + parsed_results.append(info) + + return flask.render_template('search.html', + header_playlist_names = local_playlist.get_playlist_names(), + query = query, + estimated_results = estimated_results, + estimated_pages = estimated_pages, + corrections = corrections, + results = parsed_results, + parameters_dictionary = request.args, + ) + + -- cgit v1.2.3 From 1c724f4f28804f3f8e41d222576e6fc5d7e68f75 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Fri, 21 Jun 2019 22:29:25 -0700 Subject: Search: fix corrections not working --- youtube/search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'youtube/search.py') diff --git a/youtube/search.py b/youtube/search.py index fcc352f..76a814c 100644 --- a/youtube/search.py +++ b/youtube/search.py @@ -86,7 +86,7 @@ def get_search_page(): continue if type == 'didYouMeanRenderer': renderer = renderer[type] - corrected_query_string = parameters.copy() + corrected_query_string = request.args.to_dict(flat=False) corrected_query_string['query'] = [renderer['correctedQueryEndpoint']['searchEndpoint']['query']] corrected_query_url = util.URL_ORIGIN + '/search?' + urllib.parse.urlencode(corrected_query_string, doseq=True) @@ -98,7 +98,7 @@ def get_search_page(): continue if type == 'showingResultsForRenderer': renderer = renderer[type] - no_autocorrect_query_string = parameters.copy() + no_autocorrect_query_string = request.args.to_dict(flat=False) no_autocorrect_query_string['autocorrect'] = ['0'] no_autocorrect_query_url = util.URL_ORIGIN + '/search?' + urllib.parse.urlencode(no_autocorrect_query_string, doseq=True) -- cgit v1.2.3 From 86382706fe87afc63b2757a1a133497c75ce3cd2 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Sun, 21 Jul 2019 22:28:57 -0700 Subject: Add flask Firefox search engine route --- youtube/search.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'youtube/search.py') diff --git a/youtube/search.py b/youtube/search.py index 76a814c..39a80bf 100644 --- a/youtube/search.py +++ b/youtube/search.py @@ -1,10 +1,12 @@ from youtube import util, yt_data_extract, proto, local_playlist from youtube import yt_app +import settings import json import urllib import base64 from math import ceil +import mimetypes from flask import request import flask @@ -124,4 +126,8 @@ def get_search_page(): parameters_dictionary = request.args, ) - +@yt_app.route('/opensearch.xml') +def get_search_engine_xml(): + with open("youtube/opensearch.xml", 'rb') as f: + content = f.read().replace(b'$port_number', str(settings.port_number).encode()) + return flask.Response(content, mimetype='application/xml') -- cgit v1.2.3 From e00c3cf99f06e6f0c097e019219760cf26d16cbe Mon Sep 17 00:00:00 2001 From: James Taylor Date: Tue, 23 Jul 2019 23:53:04 -0700 Subject: Remove ad-hoc response saving from code, create a debug setting for fetch_url --- youtube/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'youtube/search.py') diff --git a/youtube/search.py b/youtube/search.py index 39a80bf..e35d0cb 100644 --- a/youtube/search.py +++ b/youtube/search.py @@ -53,7 +53,7 @@ def get_search_json(query, page, autocorrect, sort, filters): 'X-YouTube-Client-Version': '2.20180418', } url += "&pbj=1&sp=" + page_number_to_sp_parameter(page, autocorrect, sort, filters).replace("=", "%3D") - content = util.fetch_url(url, headers=headers, report_text="Got search results") + content = util.fetch_url(url, headers=headers, report_text="Got search results", debug_name='search_results') info = json.loads(content) return info -- cgit v1.2.3