| 1 | #!/usr/bin/python
|
|---|
| 2 | # -*- coding: utf8 -*-
|
|---|
| 3 |
|
|---|
| 4 | # This script performs movie data lookup using the German www.ofdb.de
|
|---|
| 5 | # website.
|
|---|
| 6 | #
|
|---|
| 7 | # For more information on MythVideo's external movie lookup mechanism, see
|
|---|
| 8 | # the README file in this directory.
|
|---|
| 9 | #
|
|---|
| 10 | # This script is based on a PHP script contributed by Christian GÃŒdel and
|
|---|
| 11 | # the original ofdb.pl written by Xavier Hervy (maxpower44 AT tiscali DOT fr).
|
|---|
| 12 | #
|
|---|
| 13 | # Requirements:
|
|---|
| 14 | # PyXML
|
|---|
| 15 |
|
|---|
| 16 | import os
|
|---|
| 17 | import sys
|
|---|
| 18 | import re
|
|---|
| 19 | import urlparse
|
|---|
| 20 | import urllib
|
|---|
| 21 | import urllib2
|
|---|
| 22 | import cgi
|
|---|
| 23 | import traceback
|
|---|
| 24 | import codecs
|
|---|
| 25 | from optparse import OptionParser
|
|---|
| 26 |
|
|---|
| 27 | # This is done for Ubuntu, they are removing PyXML.
|
|---|
| 28 | alt_path = '/usr/lib/python%s/site-packages/oldxml' % sys.version[:3]
|
|---|
| 29 | if os.path.exists(alt_path):
|
|---|
| 30 | sys.path.append(alt_path)
|
|---|
| 31 |
|
|---|
| 32 | from xml.dom.ext.reader import HtmlLib
|
|---|
| 33 | from xml.dom import EMPTY_NAMESPACE
|
|---|
| 34 | from xml import xpath
|
|---|
| 35 |
|
|---|
| 36 | # The Python SGML based HTML parser is unusable (it finds
|
|---|
| 37 | # tag start/ends in the worst way possible).
|
|---|
| 38 |
|
|---|
| 39 | VERBOSE=False
|
|---|
| 40 | URL_BASE="http://www.ofdb.de"
|
|---|
| 41 | DUMP_RESPONSE=False
|
|---|
| 42 |
|
|---|
| 43 | ofdb_version = "0.3"
|
|---|
| 44 | mythtv_version = "0.21"
|
|---|
| 45 |
|
|---|
| 46 | def comment_out(str):
|
|---|
| 47 | s = str
|
|---|
| 48 | try:
|
|---|
| 49 | s = unicode(str, "utf8")
|
|---|
| 50 | except:
|
|---|
| 51 | pass
|
|---|
| 52 |
|
|---|
| 53 | print("# %s" % (s,))
|
|---|
| 54 |
|
|---|
| 55 | def debug_out(str):
|
|---|
| 56 | if VERBOSE:
|
|---|
| 57 | comment_out(str)
|
|---|
| 58 |
|
|---|
| 59 | def response_out(str):
|
|---|
| 60 | if DUMP_RESPONSE:
|
|---|
| 61 | s = str
|
|---|
| 62 | try:
|
|---|
| 63 | s = unicode(str, "utf8")
|
|---|
| 64 | except:
|
|---|
| 65 | pass
|
|---|
| 66 |
|
|---|
| 67 | print(s)
|
|---|
| 68 |
|
|---|
| 69 | def print_exception(str):
|
|---|
| 70 | for line in str.splitlines():
|
|---|
| 71 | comment_out(line)
|
|---|
| 72 |
|
|---|
| 73 | def _xmlprep(content):
|
|---|
| 74 | """Removes any HTML tags that just confuse the parser."""
|
|---|
| 75 |
|
|---|
| 76 | pat = re.compile(r'<\s*meta.*?>', re.M)
|
|---|
| 77 | ret = pat.sub('', content)
|
|---|
| 78 | pat = re.compile(r'<\s*script.*?<\s*/script\s*>', re.M | re.S)
|
|---|
| 79 | return pat.sub('', ret)
|
|---|
| 80 |
|
|---|
| 81 |
|
|---|
| 82 | def _myth_url_get(url, data = None, as_post = False):
|
|---|
| 83 | extras = ['ofdb', ofdb_version]
|
|---|
| 84 |
|
|---|
| 85 | debug_out("_myth_url_get(%s, %s, %s)" % (url, data, as_post))
|
|---|
| 86 | send_data = {}
|
|---|
| 87 | if data:
|
|---|
| 88 | send_data.update(data)
|
|---|
| 89 | dest_url = url
|
|---|
| 90 |
|
|---|
| 91 | if not as_post:
|
|---|
| 92 | # TODO: cgi.parse_qs does not handle valueless entries
|
|---|
| 93 | # so things like ?foo will fail, ?foo=val works.
|
|---|
| 94 | (scheme, netloc, path, query, frag) = urlparse.urlsplit(dest_url)
|
|---|
| 95 | send_data = {}
|
|---|
| 96 |
|
|---|
| 97 | old_qa = cgi.parse_qs(query)
|
|---|
| 98 | if old_qa:
|
|---|
| 99 | send_data.update(old_qa)
|
|---|
| 100 |
|
|---|
| 101 | if data:
|
|---|
| 102 | send_data.update(data)
|
|---|
| 103 |
|
|---|
| 104 | query = urllib.urlencode(send_data)
|
|---|
| 105 | send_data = None
|
|---|
| 106 | dest_url = urlparse.urlunsplit((scheme, netloc, path, query, frag))
|
|---|
| 107 |
|
|---|
| 108 | req = urllib2.Request(url = dest_url, headers =
|
|---|
| 109 | { 'User-Agent' : "MythTV/%s (%s)" %
|
|---|
| 110 | (mythtv_version, "; ".join(extras))})
|
|---|
| 111 |
|
|---|
| 112 | if send_data:
|
|---|
| 113 | req.add_data(urllib.urlencode(send_data))
|
|---|
| 114 |
|
|---|
| 115 | try:
|
|---|
| 116 | debug_out("Get URL '%s:%s'" % (req.get_full_url(), req.get_data()))
|
|---|
| 117 | res = urllib2.urlopen(req)
|
|---|
| 118 | content = res.read()
|
|---|
| 119 | res.close()
|
|---|
| 120 | return (res, content)
|
|---|
| 121 | except:
|
|---|
| 122 | print_exception(traceback.format_exc())
|
|---|
| 123 | return (None, None)
|
|---|
| 124 |
|
|---|
| 125 | def ofdb_url_get(url, data = None, as_post = False):
|
|---|
| 126 | (rc, content) = _myth_url_get(url, data, as_post)
|
|---|
| 127 |
|
|---|
| 128 | m = re.search(r'<\s*meta[^>]*charset\s*=\s*([^" ]+)', content, re.I)
|
|---|
| 129 | if m:
|
|---|
| 130 | charset = m.group(1)
|
|---|
| 131 | debug_out("Page charset reported as %s" % (charset))
|
|---|
| 132 | # The page lies about encoding (often using two character
|
|---|
| 133 | # encodings on the same page).
|
|---|
| 134 | content = _xmlprep(unicode(content, charset, 'replace')).encode("utf8")
|
|---|
| 135 | else:
|
|---|
| 136 | # hope it is ascii
|
|---|
| 137 | content = _xmlprep(unicode(content, errors='replace')).encode("utf8")
|
|---|
| 138 |
|
|---|
| 139 | response_out(content)
|
|---|
| 140 | return (rc, content)
|
|---|
| 141 |
|
|---|
| 142 | def search_title(title):
|
|---|
| 143 | def clean_title(t):
|
|---|
| 144 | t = urllib.unquote(t)
|
|---|
| 145 | (t, ext) = os.path.splitext(t)
|
|---|
| 146 | m = re.match("(.*)(?:[(|\[]|, The$)",t, re.I)
|
|---|
| 147 | ret = t
|
|---|
| 148 | if m:
|
|---|
| 149 | ret = m.group(1)
|
|---|
| 150 | return ret.strip().encode("utf8")
|
|---|
| 151 |
|
|---|
| 152 | try:
|
|---|
| 153 | data = {
|
|---|
| 154 | "page" : "suchergebnis",
|
|---|
| 155 | "Kat" : "DTitel",
|
|---|
| 156 | "SText" : clean_title(title)
|
|---|
| 157 | }
|
|---|
| 158 |
|
|---|
| 159 | debug_out("Starting search for title '%s'" % (title,))
|
|---|
| 160 |
|
|---|
| 161 | (rc, content) = ofdb_url_get(urlparse.urljoin(URL_BASE, "view.php"),
|
|---|
| 162 | data, True)
|
|---|
| 163 |
|
|---|
| 164 | reader = HtmlLib.Reader()
|
|---|
| 165 | doc = reader.fromString(content, charset='utf8')
|
|---|
| 166 |
|
|---|
| 167 | nodes = xpath.Evaluate("//A[starts-with(@href, 'film/')]",
|
|---|
| 168 | doc.documentElement)
|
|---|
| 169 |
|
|---|
| 170 | title_matches = []
|
|---|
| 171 | uid_match = re.compile('/(\d+),.*', re.I)
|
|---|
| 172 | for title in nodes:
|
|---|
| 173 | rm = uid_match.search(title.getAttributeNS(EMPTY_NAMESPACE, 'href'))
|
|---|
| 174 | if rm:
|
|---|
| 175 | title_matches.append((rm.group(1), title.firstChild.nodeValue))
|
|---|
| 176 |
|
|---|
| 177 | for id, title in title_matches:
|
|---|
| 178 | print("%s:%s" % (id, title.strip()))
|
|---|
| 179 | except:
|
|---|
| 180 | print_exception(traceback.format_exc())
|
|---|
| 181 |
|
|---|
| 182 | def get_ofdb_doc(uid, context):
|
|---|
| 183 | """Returns the OFDb film page as an XML document."""
|
|---|
| 184 | debug_out("Starting search for %s '%s'" % (context, uid))
|
|---|
| 185 |
|
|---|
| 186 | (rc, content) = ofdb_url_get(urlparse.urljoin(URL_BASE,
|
|---|
| 187 | "film/%s," % (uid.encode("utf8"),)))
|
|---|
| 188 |
|
|---|
| 189 | reader = HtmlLib.Reader()
|
|---|
| 190 | return reader.fromString(content, charset='utf8')
|
|---|
| 191 |
|
|---|
| 192 | class NoIMDBURL(Exception):
|
|---|
| 193 | pass
|
|---|
| 194 |
|
|---|
| 195 | def search_data(uid, rating_country):
|
|---|
| 196 | def possible_error(path):
|
|---|
| 197 | comment_out("Warning: expected to find content at '%s', site format " \
|
|---|
| 198 | "may have changed, look for a new version of this script." %
|
|---|
| 199 | (path,))
|
|---|
| 200 |
|
|---|
| 201 | def single_value(doc, path):
|
|---|
| 202 | nodes = xpath.Evaluate(path, doc)
|
|---|
| 203 | if len(nodes):
|
|---|
| 204 | return nodes[0].firstChild.nodeValue.strip()
|
|---|
| 205 | possible_error(path)
|
|---|
| 206 | return ""
|
|---|
| 207 |
|
|---|
| 208 | def attr_value(doc, path, attrname):
|
|---|
| 209 | nodes = xpath.Evaluate(path, doc)
|
|---|
| 210 | if len(nodes):
|
|---|
| 211 | return nodes[0].getAttributeNS(EMPTY_NAMESPACE, attrname).strip()
|
|---|
| 212 | possible_error(path)
|
|---|
| 213 | return ""
|
|---|
| 214 |
|
|---|
| 215 | def multi_value(doc, path):
|
|---|
| 216 | ret = []
|
|---|
| 217 | nodes = xpath.Evaluate(path, doc)
|
|---|
| 218 | if len(nodes):
|
|---|
| 219 | for i in nodes:
|
|---|
| 220 | ret.append(i.firstChild.nodeValue.strip())
|
|---|
| 221 | return ret
|
|---|
| 222 | possible_error(path)
|
|---|
| 223 | return ""
|
|---|
| 224 |
|
|---|
| 225 | def direct_value(doc, path):
|
|---|
| 226 | nodes = xpath.Evaluate(path, doc)
|
|---|
| 227 | if len(nodes):
|
|---|
| 228 | return nodes[0].nodeValue.strip()
|
|---|
| 229 | possible_error(path)
|
|---|
| 230 | return ""
|
|---|
| 231 |
|
|---|
| 232 | def all_text_children(doc, path):
|
|---|
| 233 | nodes = xpath.Evaluate(path, doc)
|
|---|
| 234 | if len(nodes):
|
|---|
| 235 | ret = []
|
|---|
| 236 | for n in nodes:
|
|---|
| 237 | for c in n.childNodes:
|
|---|
| 238 | if c.nodeType == c.TEXT_NODE:
|
|---|
| 239 | ret.append(c.nodeValue.strip())
|
|---|
| 240 | return " ".join(ret)
|
|---|
| 241 | possible_error(path)
|
|---|
| 242 | return ""
|
|---|
| 243 |
|
|---|
| 244 | try:
|
|---|
| 245 | doc = get_ofdb_doc(uid, "data")
|
|---|
| 246 |
|
|---|
| 247 | #TODO: Add details from tmdb
|
|---|
| 248 | data = {'title' : '',
|
|---|
| 249 | 'countries' : '',
|
|---|
| 250 | 'year' : '',
|
|---|
| 251 | 'directors' : '',
|
|---|
| 252 | 'cast' : '',
|
|---|
| 253 | 'genre' : '',
|
|---|
| 254 | 'user_rating' : '',
|
|---|
| 255 | # 'movie_rating' : '',
|
|---|
| 256 | 'plot' : '',
|
|---|
| 257 | # 'release_date' : '',
|
|---|
| 258 | # 'runtime' : '',
|
|---|
| 259 | # 'writers' : '',
|
|---|
| 260 | }
|
|---|
| 261 |
|
|---|
| 262 | data['title'] = single_value(doc.documentElement,
|
|---|
| 263 | "//TD[@width='99%']/H2/FONT[@size='3']/B")
|
|---|
| 264 | data['countries'] = ",".join(multi_value(doc.documentElement,
|
|---|
| 265 | "//A[starts-with(@href, 'view.php?page=blaettern&Kat=Land&')]"))
|
|---|
| 266 | data['year'] = single_value(doc.documentElement,
|
|---|
| 267 | "//A[starts-with(@href, 'view.php?page=blaettern&Kat=Jahr&')]")
|
|---|
| 268 | data['directors'] = ",".join(multi_value(doc.documentElement,
|
|---|
| 269 | "//TD[@width='99%']/TABLE/TR[4]/TD[3]//A[starts-with(@href, " \
|
|---|
| 270 | "'view.php?page=liste')]"))
|
|---|
| 271 | data['cast'] = ",".join(multi_value(doc.documentElement,
|
|---|
| 272 | "//TD[@width='99%']/TABLE/TR[5]/TD[3]//A[starts-with(@href, " \
|
|---|
| 273 | "'view.php?page=liste')]"))
|
|---|
| 274 | data['genre'] = ",".join(multi_value(doc.documentElement,
|
|---|
| 275 | "//A[starts-with(@href, 'view.php?page=genre&Genre=')]"))
|
|---|
| 276 | data['user_rating'] = attr_value(doc.documentElement,
|
|---|
| 277 | "//IMG[@src='images/design3/notenspalte.png']", "alt")
|
|---|
| 278 |
|
|---|
| 279 | tmp_sid = attr_value(doc.documentElement,
|
|---|
| 280 | "//A[starts-with(@href, 'plot/')]", "href")
|
|---|
| 281 |
|
|---|
| 282 | sid_match = re.search("/(\d+,\d+,.*)", tmp_sid, re.I)
|
|---|
| 283 | sid = None
|
|---|
| 284 | if sid_match:
|
|---|
| 285 | sid = sid_match.group(1)
|
|---|
| 286 |
|
|---|
| 287 | debug_out("Looking for plot...")
|
|---|
| 288 | (rc, content) = ofdb_url_get(urlparse.urljoin(URL_BASE,
|
|---|
| 289 | "plot/%s" % sid.encode("utf8")))
|
|---|
| 290 |
|
|---|
| 291 | reader = HtmlLib.Reader()
|
|---|
| 292 | doc = reader.fromString(content, charset='utf8')
|
|---|
| 293 |
|
|---|
| 294 | data['plot'] = unicode(all_text_children(doc.documentElement,
|
|---|
| 295 | "//FONT[@class='Blocksatz']"))
|
|---|
| 296 |
|
|---|
| 297 | #ReleaseDate:%(release_date)s
|
|---|
| 298 | #MovieRating:%(movie_rating)s
|
|---|
| 299 | #Runtime:%(runtime)s
|
|---|
| 300 | #Writers:%(writers)s
|
|---|
| 301 | print("""\
|
|---|
| 302 | Title:%(title)s
|
|---|
| 303 | Year:%(year)s
|
|---|
| 304 | Director:%(directors)s
|
|---|
| 305 | Plot:%(plot)s
|
|---|
| 306 | UserRating:%(user_rating)s
|
|---|
| 307 | Cast:%(cast)s
|
|---|
| 308 | Genres:%(genre)s
|
|---|
| 309 | Countries:%(countries)s
|
|---|
| 310 | """ % data)
|
|---|
| 311 |
|
|---|
| 312 | except:
|
|---|
| 313 | print_exception(traceback.format_exc())
|
|---|
| 314 |
|
|---|
| 315 | def search_poster(uid):
|
|---|
| 316 | try:
|
|---|
| 317 | debug_out("Looking for posters...")
|
|---|
| 318 | poster_urls = []
|
|---|
| 319 | ofdoc = get_ofdb_doc(uid, "poster")
|
|---|
| 320 |
|
|---|
| 321 | nodes = xpath.Evaluate("//IMG[starts-with(@src, 'http://img.ofdb.de/film/')]",
|
|---|
| 322 | ofdoc.documentElement)
|
|---|
| 323 | for node in nodes:
|
|---|
| 324 | poster_urls.append(node.getAttributeNS(EMPTY_NAMESPACE, 'src'))
|
|---|
| 325 |
|
|---|
| 326 | for p in poster_urls:
|
|---|
| 327 | print(p)
|
|---|
| 328 | except:
|
|---|
| 329 | print_exception(traceback.format_exc())
|
|---|
| 330 |
|
|---|
| 331 | def main():
|
|---|
| 332 | parser = OptionParser(usage="""\
|
|---|
| 333 | Usage: %prog [-M TITLE | -D UID [-R COUNTRY[,COUNTRY]] | -P UID]
|
|---|
| 334 | """, version="%%prog %s" % (ofdb_version))
|
|---|
| 335 | parser.add_option("-M", "--title", type="string", dest="title_search",
|
|---|
| 336 | metavar="TITLE", help="Search for TITLE")
|
|---|
| 337 | parser.add_option("-D", "--data", type="string", dest="data_search",
|
|---|
| 338 | metavar="UID", help="Search for video data for UID")
|
|---|
| 339 | parser.add_option("-R", "--rating-country", type="string",
|
|---|
| 340 | dest="ratings_from", metavar="COUNTRY",
|
|---|
| 341 | help="When retrieving data, use ratings from COUNTRY")
|
|---|
| 342 | parser.add_option("-P", "--poster", type="string", dest="poster_search",
|
|---|
| 343 | metavar="UID", help="Search for images associated with UID")
|
|---|
| 344 | parser.add_option("-d", "--debug", action="store_true", dest="verbose",
|
|---|
| 345 | default=False, help="Display debug information")
|
|---|
| 346 | parser.add_option("-r", "--dump-response", action="store_true",
|
|---|
| 347 | dest="dump_response", default=False,
|
|---|
| 348 | help="Output the raw response")
|
|---|
| 349 |
|
|---|
| 350 | (options, args) = parser.parse_args()
|
|---|
| 351 |
|
|---|
| 352 | global VERBOSE, DUMP_RESPONSE
|
|---|
| 353 | VERBOSE = options.verbose
|
|---|
| 354 | DUMP_RESPONSE = options.dump_response
|
|---|
| 355 |
|
|---|
| 356 | if options.title_search:
|
|---|
| 357 | search_title(unicode(options.title_search, "utf8"))
|
|---|
| 358 | elif options.data_search:
|
|---|
| 359 | rf = options.ratings_from
|
|---|
| 360 | if rf:
|
|---|
| 361 | rf = unicode(rf, "utf8")
|
|---|
| 362 | search_data(unicode(options.data_search, "utf8"), rf)
|
|---|
| 363 | elif options.poster_search:
|
|---|
| 364 | search_poster(unicode(options.poster_search, "utf8"))
|
|---|
| 365 | else:
|
|---|
| 366 | parser.print_usage()
|
|---|
| 367 | sys.exit(1)
|
|---|
| 368 |
|
|---|
| 369 | if __name__ == '__main__':
|
|---|
| 370 | try:
|
|---|
| 371 | codecinfo = codecs.lookup('utf8')
|
|---|
| 372 |
|
|---|
| 373 | u2utf8 = codecinfo.streamwriter(sys.stdout)
|
|---|
| 374 | sys.stdout = u2utf8
|
|---|
| 375 |
|
|---|
| 376 | main()
|
|---|
| 377 | except SystemExit:
|
|---|
| 378 | pass
|
|---|
| 379 | except:
|
|---|
| 380 | print_exception(traceback.format_exc())
|
|---|
| 381 |
|
|---|
| 382 | # vim: ts=4 sw=4:
|
|---|
| 383 |
|
|---|
