Index: mythtv/programs/mythfilldatabase/filldata.cpp =================================================================== --- mythtv/programs/mythfilldatabase/filldata.cpp (revision 14158) +++ mythtv/programs/mythfilldatabase/filldata.cpp (working copy) @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -54,6 +55,7 @@ bool isNorthAmerica = false; bool isJapan = false; bool interrupted = false; +bool endofdata = false; bool refresh_today = false; bool refresh_tomorrow = true; bool refresh_second = false; @@ -212,6 +214,10 @@ QString userid; QString password; QString lineupid; + bool xmltvgrabber_baseline; + bool xmltvgrabber_manualconfig; + bool xmltvgrabber_cache; + QString xmltvgrabber_prefmethod; }; @@ -2505,7 +2511,15 @@ return false; handleChannels(id, &chanlist); - handlePrograms(id, &proglist); + if (proglist.count() == 0) { + VERBOSE(VB_GENERAL, + QString("No programs found in data.")); + endofdata = true; + } + else { + handlePrograms(id, &proglist); + } + return true; } @@ -2533,10 +2547,10 @@ return grabDDData(source, offset, *qCurrentDate, DD_ZAP2IT); else if (xmltv_grabber == "technovera") { - VERBOSE(VB_ALL, "This grabber is no longer supported"); + VERBOSE(VB_IMPORTANT, "The technovera grabber is no longer supported"); exit(FILLDB_EXIT_INVALID_CMDLINE); } - + char tempfilename[] = "/tmp/mythXXXXXX"; if (mkstemp(tempfilename) == -1) { @@ -2551,127 +2565,32 @@ QString home = QDir::homeDirPath(); QString configfile = QString("%1/%2.xmltv").arg(MythContext::GetConfDir()) .arg(source.name); - QString command; - if (xmltv_grabber == "tv_grab_uk") - command.sprintf("nice %s --days 7 --config-file '%s' --output %s", - xmltv_grabber.ascii(), configfile.ascii(), - filename.ascii()); - else if (xmltv_grabber == "tv_grab_uk_rt") - command.sprintf("nice %s --days 14 --config-file '%s' --output %s", - xmltv_grabber.ascii(), - configfile.ascii(), filename.ascii()); - else if (xmltv_grabber == "tv_grab_au") - command.sprintf("nice %s --days 7 --config-file '%s' --output %s", - xmltv_grabber.ascii(), configfile.ascii(), - filename.ascii()); - else if (xmltv_grabber == "tv_grab_de_tvtoday") - command.sprintf("nice %s --slow --days 1 --config-file '%s' --offset %d --output %s", - xmltv_grabber.ascii(), configfile.ascii(), - offset, filename.ascii()); - else if (xmltv_grabber == "tv_grab_fr") - command.sprintf("nice %s --days 7 --config-file '%s' --output %s", - xmltv_grabber.ascii(), configfile.ascii(), - filename.ascii()); - else if (xmltv_grabber == "tv_grab_nl") - command.sprintf("nice %s --config-file '%s' --output %s", - xmltv_grabber.ascii(), configfile.ascii(), - filename.ascii()); - else if (xmltv_grabber == "tv_grab_fi") - // Use the default of 10 days for Finland's grabber - command.sprintf("nice %s --config-file '%s' --output %s", - xmltv_grabber.ascii(), configfile.ascii(), - filename.ascii()); - else if (xmltv_grabber == "tv_grab_es" || - xmltv_grabber == "tv_grab_es_laguiatv") - // Use fixed interval of 3 days for Spanish grabber - command.sprintf("nice %s --days=4 --config-file '%s' --output %s", - xmltv_grabber.ascii(), - configfile.ascii(), filename.ascii()); - else if (xmltv_grabber == "tv_grab_jp") + QString command = QString("nice %1 --config-file '%2' --output %3") + .arg(xmltv_grabber.ascii()) + .arg(configfile.ascii()) + .arg(filename.ascii()); + + // The one concession to grabber specific behaviour. + // Will be removed when the grabber allows. + if (xmltv_grabber == "tv_grab_jp") { - // Use fixed interval of 7 days for Japanese grabber - command.sprintf("nice %s --days 7 --enable-readstr --config-file '%s' --output %s", - xmltv_grabber.ascii(), configfile.ascii(), - filename.ascii()); - isJapan = true; + command += QString(" --enable-readstr"); + isJapan = true; } - else if (xmltv_grabber == "tv_grab_no") - command.sprintf("nice %s --days 1 --offset %d --config-file '%s' --output %s", - xmltv_grabber.ascii(), offset, configfile.ascii(), - filename.ascii()); - else if (xmltv_grabber == "tv_grab_se_swedb") - command.sprintf("nice %s --days 1 --offset %d --config-file '%s' --output %s", - xmltv_grabber.ascii(), offset, configfile.ascii(), - filename.ascii()); - else if (xmltv_grabber == "tv_grab_dk") - // Use fixed interval of 7 days for Danish grabber - command.sprintf("nice %s --days 7 --config-file '%s' --output %s", - xmltv_grabber.ascii(), configfile.ascii(), - filename.ascii()); - else if (xmltv_grabber == "tv_grab_pt") - // Use fixed interval of 3 days for Portuguese grabber - command.sprintf("nice %s --days=4 --config-file '%s' --output %s", - xmltv_grabber.ascii(), - configfile.ascii(), filename.ascii()); - else if (xmltv_grabber == "tv_grab_be_tvb") - command.sprintf("nice %s --days 1 --offset %d --config-file '%s' --output %s", - xmltv_grabber.ascii(), offset, configfile.ascii(), - filename.ascii()); - else if (xmltv_grabber == "tv_grab_be_tlm") - command.sprintf("nice %s --days 1 --offset %d --config-file '%s' --output %s", - xmltv_grabber.ascii(), offset, configfile.ascii(), - filename.ascii()); - else if (xmltv_grabber == "tv_grab_ee") - // Estonian grabber returns all known data by default - command.sprintf("nice %s --output %s", - xmltv_grabber.ascii(), - filename.ascii()); - else if (xmltv_grabber == "tv_grab_il") - // Israeli grabber returns all known data by default - command.sprintf("nice %s --config-file '%s' --output %s", - xmltv_grabber.ascii(), configfile.ascii(), - filename.ascii()); - else if (xmltv_grabber == "tv_grab_ru") - // Russian grabber returns all known data by default - command.sprintf("nice %s --config-file '%s' --output %s", - xmltv_grabber.ascii(), configfile.ascii(), - filename.ascii()); - else + else if (source.xmltvgrabber_prefmethod != "allatonce") { - isNorthAmerica = true; - command.sprintf("nice %s --days 1 --offset %d --config-file '%s' " - "--output %s", xmltv_grabber.ascii(), - offset, configfile.ascii(), filename.ascii()); + // XMLTV Docs don't recommend grabbing one day at a + // time but the current myth code is heavily geared + // that way so until it is re-written behave as + // we always have done. + command += QString(" --days 1 --offset %1").arg(offset); } - if (((print_verbose_messages & VB_GENERAL) == 0) && - (xmltv_grabber == "tv_grab_na" || - xmltv_grabber == "tv_grab_de_tvtoday" || - xmltv_grabber == "tv_grab_fi" || - xmltv_grabber == "tv_grab_es" || - xmltv_grabber == "tv_grab_es_laguiatv" || - xmltv_grabber == "tv_grab_se_swedb" || - xmltv_grabber == "tv_grab_no" || - xmltv_grabber == "tv_grab_dk" || - xmltv_grabber == "tv_grab_uk" || - xmltv_grabber == "tv_grab_uk_rt" || - xmltv_grabber == "tv_grab_nl" || - xmltv_grabber == "tv_grab_fr" || - xmltv_grabber == "tv_grab_fi" || - xmltv_grabber == "tv_grab_jp" || - xmltv_grabber == "tv_grab_pt" || - xmltv_grabber == "tv_grab_be_tvb" || - xmltv_grabber == "tv_grab_be_tlm" || - xmltv_grabber == "tv_grab_ee" || - xmltv_grabber == "tv_grab_ru")) - command += " --quiet"; - - command += graboptions; - VERBOSE(VB_GENERAL, - "----------------- Start of XMLTV output -----------------"); + if (! (print_verbose_messages & VB_GENERAL)) + command += " --quiet"; QDateTime qdtNow = QDateTime::currentDateTime(); MSqlQuery query(MSqlQuery::InitCon()); @@ -2685,10 +2604,18 @@ "WHERE value='mythfilldatabaseLastRunStatus'") .arg(status)); + VERBOSE(VB_GENERAL, QString("Grabber Command: %1").arg(command)); + + VERBOSE(VB_GENERAL, + "----------------- Start of XMLTV output -----------------"); + int systemcall_status = system(command.ascii()); bool succeeded = WIFEXITED(systemcall_status) && WEXITSTATUS(systemcall_status) == 0; + VERBOSE(VB_GENERAL, + "------------------ End of XMLTV output ------------------"); + qdtNow = QDateTime::currentDateTime(); query.exec(QString("UPDATE settings SET data ='%1' " "WHERE value='mythfilldatabaseLastRunEnd'") @@ -2704,13 +2631,13 @@ query.exec(QString("UPDATE settings SET data ='%1' " "WHERE value='mythfilldatabaseLastRunStatus'") .arg(status)); + + VERBOSE(VB_GENERAL, status); + if (WIFSIGNALED(systemcall_status) && (WTERMSIG(systemcall_status) == SIGINT || WTERMSIG(systemcall_status) == SIGQUIT)) interrupted = true; } - - VERBOSE(VB_GENERAL, - "------------------ End of XMLTV output ------------------"); grabDataFromFile(source.id, filename); @@ -2834,45 +2761,194 @@ int failures = 0; int externally_handled = 0; int total_sources = sourcelist.size(); + int source_channels = 0; - query.exec(QString("SELECT MAX(endtime) FROM program WHERE manualid=0;")); - if (query.isActive() && query.size() > 0) - { - query.next(); - - if (!query.isNull(0)) - GuideDataBefore = QDateTime::fromString(query.value(0).toString(), - Qt::ISODate); - } - QString sidStr = QString("Updating source #%1 (%2) with grabber %3"); need_post_grab_proc = false; + int nonewdata = 0; for (it = sourcelist.begin(); it != sourcelist.end(); ++it) { + + query.prepare("SELECT MAX(endtime) FROM program p LEFT JOIN channel c " + "ON p.chanid=c.chanid WHERE c.sourceid= :SRCID " + "AND manualid = 0;"); + query.bindValue(":SRCID", (*it).id); + query.exec(); + if (query.isActive() && query.size() > 0) + { + query.next(); + + if (!query.isNull(0)) + GuideDataBefore = QDateTime::fromString(query.value(0).toString(), + Qt::ISODate); + } + channel_update_run = false; + endofdata = false; + + QString xmltv_grabber = (*it).xmltvgrabber; + + if (xmltv_grabber == "eitonly") + { + VERBOSE(VB_IMPORTANT, "Source configured to use only the " + "broadcasted guide data. Skipping."); + externally_handled++; + query.exec(QString("UPDATE settings SET data ='%1' " + "WHERE value='mythfilldatabaseLastRunStart' OR " + "value = 'mythfilldatabaseLastRunEnd'") + .arg(QDateTime::currentDateTime().toString("yyyy-MM-dd hh:mm"))); + continue; + } + else if (xmltv_grabber == "/bin/true" || + xmltv_grabber == "none" || + xmltv_grabber == "") + { + VERBOSE(VB_IMPORTANT, + "Source configured with no grabber. Nothing to do."); + externally_handled++; + query.exec(QString("UPDATE settings SET data ='%1' " + "WHERE value='mythfilldatabaseLastRunStart' OR " + "value = 'mythfilldatabaseLastRunEnd'") + .arg(QDateTime::currentDateTime().toString("yyyy-MM-dd hh:mm"))); + continue; + } + VERBOSE(VB_GENERAL, sidStr.arg((*it).id) .arg((*it).name) - .arg((*it).xmltvgrabber)); + .arg(xmltv_grabber)); - QString xmltv_grabber = (*it).xmltvgrabber; - need_post_grab_proc |= (xmltv_grabber != "datadirect"); + query.prepare( + "SELECT COUNT(chanid) FROM channel WHERE sourceid = " + ":SRCID AND xmltvid != ''"); + query.bindValue(":SRCID", (*it).id); + query.exec(); - if (xmltv_grabber == "tv_grab_uk" || xmltv_grabber == "tv_grab_uk_rt" || - xmltv_grabber == "tv_grab_fi" || xmltv_grabber == "tv_grab_es" || - xmltv_grabber == "tv_grab_es_laguiatv" || - xmltv_grabber == "tv_grab_nl" || xmltv_grabber == "tv_grab_au" || - xmltv_grabber == "tv_grab_fr" || xmltv_grabber == "tv_grab_jp" || - xmltv_grabber == "tv_grab_pt" || xmltv_grabber == "tv_grab_ee" || - xmltv_grabber == "tv_grab_dk") + if (query.isActive() && query.size() > 0) { + query.next(); + source_channels = query.value(0).toInt(); + if (source_channels > 0) + { + VERBOSE(VB_GENERAL, QString("Found %1 channels for " + "source %2 which use grabber") + .arg(source_channels).arg((*it).id)); + } + else + { + VERBOSE(VB_GENERAL, QString("No channels are " + "configured to use grabber.")); + } + } + else { + source_channels = 0; + VERBOSE(VB_GENERAL, + QString("Can't get a channel count for source id %1") + .arg((*it).id)); + } + + bool hasprefmethod = false; + + if (xmltv_grabber != "datadirect") { + + QProcess grabber_capabilities_proc(xmltv_grabber); + grabber_capabilities_proc.addArgument(QString("--capabilities")); + if ( grabber_capabilities_proc.start() ) + { + + int i=0; + // Assume it shouldn't take more than 10 seconds + // Broken versions of QT cause QProcess::start + // and QProcess::isRunning to return true even + // when the executable doesn't exist + while (grabber_capabilities_proc.isRunning() && i < 100) + { + usleep(100000); + ++i; + } + + if (grabber_capabilities_proc.normalExit()) + { + QString capabilites = ""; + + while (grabber_capabilities_proc.canReadLineStdout()) + { + QString capability + = grabber_capabilities_proc.readLineStdout(); + capabilites += capability + " "; + + if (capability == "baseline") + (*it).xmltvgrabber_baseline = true; + + if (capability == "manualconfig") + (*it).xmltvgrabber_manualconfig = true; + + if (capability == "cache") + (*it).xmltvgrabber_cache = true; + + if (capability == "preferredmethod") + hasprefmethod = true; + } + + VERBOSE(VB_GENERAL, QString("Grabber has capabilities: %1") + .arg(capabilites)); + } + else { + VERBOSE(VB_IMPORTANT, "%1 --capabilities failed or we " + "timed out waiting. You may need to upgrade your " + "xmltv grabber"); + } + } + else { + QString error = grabber_capabilities_proc.readLineStdout(); + VERBOSE(VB_IMPORTANT, QString("Failed to run %1 " + "--capabilities").arg(xmltv_grabber)); + } + } + + + if (hasprefmethod) { - // These don't support the --offset option, so just grab the max. - // TODO: tv_grab_fi/dk/is seems to support --offset, maybe more. Needs verification. - if (!grabData(*it, 0)) - ++failures; + + QProcess grabber_method_proc(xmltv_grabber); + grabber_method_proc.addArgument("--preferredmethod"); + if ( grabber_method_proc.start() ) + { + int i=0; + // Assume it shouldn't take more than 10 seconds + // Broken versions of QT cause QProcess::start + // and QProcess::isRunning to return true even + // when the executable doesn't exist + while (grabber_method_proc.isRunning() && i < 100) + { + usleep(100000); + ++i; + } + + if (grabber_method_proc.normalExit()) + { + (*it).xmltvgrabber_prefmethod = + grabber_method_proc.readLineStdout(); + } + else { + VERBOSE(VB_IMPORTANT, "%1 --preferredmethod failed or we " + "timed out waiting. You may need to upgrade your " + "xmltv grabber"); + } + + VERBOSE(VB_GENERAL, QString("Grabber prefers method: %1") + .arg((*it).xmltvgrabber_prefmethod)); + } + else { + QString error = grabber_method_proc.readLineStdout(); + VERBOSE(VB_IMPORTANT, QString("Failed to run %1 --preferredmethod") + .arg(xmltv_grabber)); + } } - else if ((xmltv_grabber == "datadirect") && dd_grab_all) + + need_post_grab_proc |= (xmltv_grabber != "datadirect"); + + if ((xmltv_grabber == "datadirect") && dd_grab_all) { if (only_update_channels) DataDirectUpdateChannels(*it); @@ -2882,40 +2958,24 @@ grabData(*it, 0, &qCurrentDate); } } - else if (xmltv_grabber == "datadirect" || - xmltv_grabber == "tv_grab_se_swedb" || - xmltv_grabber == "tv_grab_no" || - xmltv_grabber == "tv_grab_de_tvtoday" || - xmltv_grabber == "tv_grab_be_tvb" || - xmltv_grabber == "tv_grab_be_tlm" || - xmltv_grabber == "tv_grab_is" || - xmltv_grabber == "tv_grab_br" || - xmltv_grabber == "tv_grab_cz" || - xmltv_grabber == "tv_grab_ru") + else if ((*it).xmltvgrabber_prefmethod == "allatonce") { - // Grabbers supporting the --offset option + if (!grabData(*it, 0)) + ++failures; + } + else if ((*it).xmltvgrabber_baseline || xmltv_grabber == "datadirect") + { - if (xmltv_grabber == "tv_grab_no") - listing_wrap_offset = 6 * 3600; - QDate qCurrentDate = QDate::currentDate(); - int grabdays = 9; + // We'll keep grabbing until it returns nothing + // Max days currently supported is 21 + int grabdays = 21; - // Grab different amount of days for the different grabbers, - // often decided by the person maintaining the grabbers. if (maxDays > 0) // passed with --max-days grabdays = maxDays; else if (xmltv_grabber == "datadirect") grabdays = 14; - else if (xmltv_grabber == "tv_grab_se_swedb") - grabdays = 10; - else if (xmltv_grabber == "tv_grab_no" || - xmltv_grabber == "tv_grab_de_tvtoday") - grabdays = 7; - else if (xmltv_grabber == "tv_grab_be_tvb" || - xmltv_grabber == "tv_grab_be_tlm") - grabdays = 5; grabdays = (only_update_channels) ? 1 : grabdays; @@ -2961,17 +3021,19 @@ (i == 2 && refresh_second)) { // Always refresh if the user specified today/tomorrow/second. - download_needed = true; + if (refresh_today) { + VERBOSE(VB_GENERAL, + "Data Refresh needed because user specified --refresh-today"); + } + else if (refresh_second) { + VERBOSE(VB_GENERAL, + "Data Refresh needed because user specified --refresh-second"); + } + else { + VERBOSE(VB_GENERAL, + "Data Refresh always needed for tomorrow"); + } } - else if (xmltv_grabber == "tv_grab_se_swedb") - { - // Since tv_grab_se_swedb handles caching internally, - // let it do its job and always grab new data. - VERBOSE(VB_GENERAL, - "Data Refresh needed because the grabber relies on " - "internal caching."); - download_needed = true; - } else { // Check to see if we already downloaded data for this date. @@ -2987,9 +3049,9 @@ "INTERVAL '%1' DAY), INTERVAL '18' HOUR) " " AND starttime < DATE_ADD(CURRENT_DATE(), " "INTERVAL '%2' DAY) " - "WHERE c.sourceid = %3 " + "WHERE c.sourceid = %3 AND c.xmltvid != '' " "GROUP BY c.chanid;"; - + if (query.exec(querystr.arg(i-1).arg(i).arg((*it).id)) && query.isActive()) { @@ -3086,7 +3148,7 @@ "offset day %2.").arg(i-1).arg(i)); download_needed = true; } - } + } if (download_needed) { @@ -3100,6 +3162,13 @@ break; } } + + if (endofdata) { + VERBOSE(VB_GENERAL, + QString("Grabber is no longer returning program data, finishing")); + break; + } + } else { @@ -3109,61 +3178,51 @@ } } } - else if (xmltv_grabber == "eitonly") - { - VERBOSE(VB_IMPORTANT, "Source configured to use only the " - "broadcasted guide data. Skipping."); - externally_handled++; - query.exec(QString("UPDATE settings SET data ='%1' " - "WHERE value='mythfilldatabaseLastRunStart' OR " - "value = 'mythfilldatabaseLastRunEnd'") - .arg(QDateTime::currentDateTime().toString("yyyy-MM-dd hh:mm"))); - } - else if (xmltv_grabber == "/bin/true" || - xmltv_grabber == "none" || - xmltv_grabber == "") - { - VERBOSE(VB_IMPORTANT, - "Source configured with no grabber. Nothing to do."); - externally_handled++; - query.exec(QString("UPDATE settings SET data ='%1' " - "WHERE value='mythfilldatabaseLastRunStart' OR " - "value = 'mythfilldatabaseLastRunEnd'") - .arg(QDateTime::currentDateTime().toString("yyyy-MM-dd hh:mm"))); - } else { VERBOSE(VB_IMPORTANT, QString("Grabbing XMLTV data using ") + xmltv_grabber + - " is not verified as working."); + " is not supported. You may need to upgrade to" + " the latest version of XMLTV."); } if (interrupted) { break; } + + query.prepare("SELECT MAX(endtime) FROM program p LEFT JOIN channel c " + "ON p.chanid=c.chanid WHERE c.sourceid= :SRCID " + "AND manualid = 0;"); + query.bindValue(":SRCID", (*it).id); + query.exec(); + if (query.isActive() && query.size() > 0) + { + query.next(); + + if (!query.isNull(0)) + GuideDataAfter = QDateTime::fromString(query.value(0).toString(), + Qt::ISODate); + } + + if (GuideDataAfter == GuideDataBefore) + { + nonewdata++; + } } if (only_update_channels && !need_post_grab_proc) return true; - query.exec(QString("SELECT MAX(endtime) FROM program WHERE manualid=0;")); - if (query.isActive() && query.size() > 0) - { - query.next(); - - if (!query.isNull(0)) - GuideDataAfter = QDateTime::fromString(query.value(0).toString(), - Qt::ISODate); - } - if (failures == 0) { - if ((GuideDataAfter == GuideDataBefore) && + if (nonewdata > 0 && (total_sources != externally_handled)) - status = "mythfilldatabase ran, but did not insert " - "any new data into the Guide. This can indicate a " - "potential grabber failure."; + status = QString("mythfilldatabase ran, but did not insert " + "any new data into the Guide for %1 of %2 sources. " + "This can indicate a potential grabber failure.") + .arg(nonewdata) + .arg(total_sources); else status = "Successful."; @@ -3694,7 +3753,7 @@ cout << "--refresh-today\n"; cout << "--refresh-second\n"; cout << "--refresh-all\n"; - cout << " (Only valid for grabbers: DataDirect, se_swedb, no, ee, de_tvtoday)\n"; + cout << " (Only valid for selected grabbers: e.g. DataDirect)\n"; cout << " Force a refresh today or two days (or every day) from now,\n"; cout << " to catch the latest changes\n"; cout << "--dont-refresh-tomorrow\n"; @@ -3843,7 +3902,7 @@ "FROM videosource ") + where + QString(" ORDER BY sourceid;"); sourcequery.exec(querystr); - + if (sourcequery.isActive()) { if (sourcequery.size() > 0) @@ -3851,7 +3910,7 @@ while (sourcequery.next()) { Source newsource; - + newsource.id = sourcequery.value(0).toInt(); newsource.name = sourcequery.value(1).toString(); newsource.xmltvgrabber = sourcequery.value(2).toString(); @@ -3859,6 +3918,11 @@ newsource.password = sourcequery.value(4).toString(); newsource.lineupid = sourcequery.value(5).toString(); + newsource.xmltvgrabber_baseline = false; + newsource.xmltvgrabber_manualconfig = false; + newsource.xmltvgrabber_cache = false; + newsource.xmltvgrabber_prefmethod = ""; + sourcelist.append(newsource); if (newsource.xmltvgrabber == "datadirect") usingDataDirect = true;