Index: programs/mythbackend/scheduler.h =================================================================== --- programs/mythbackend/scheduler.h (revision 14501) +++ programs/mythbackend/scheduler.h (working copy) @@ -2,6 +2,7 @@ #define SCHEDULER_H_ // C++ headers +#include #include #include using namespace std; @@ -35,7 +36,10 @@ typedef RecList::iterator RecIter; typedef vector InputGroupList; +typedef QMap CardListMap; +typedef multimap ConflictMap; + class Scheduler : public QObject { public: @@ -99,11 +103,23 @@ void PruneOverlaps(void); void BuildListMaps(void); void ClearListMaps(void); + + bool IsConflicting(const ProgramInfo *p, const ProgramInfo *q, + bool openEnd) const; + + bool IsSameProgram(const ProgramInfo *a, const ProgramInfo *b) const; + bool IsSameTimeslot(const ProgramInfo *a, const ProgramInfo *b) const; + bool FindNextConflict(const RecList &cardlist, const ProgramInfo *p, RecConstIter &iter, bool openEnd = false) const; - const ProgramInfo *FindConflict(const QMap &reclists, - const ProgramInfo *p, bool openEnd = false) const; + + ProgramInfo *FindAnyConflict( + ConflictMap &map, const ProgramInfo *p, bool openend) const; + + RecList FindAllConflict( + ConflictMap &map, const ProgramInfo *p, bool openend) const; + void MarkOtherShowings(ProgramInfo *p); void MarkShowingsList(RecList &showinglist, ProgramInfo *p); void BackupRecStatus(void); @@ -137,7 +153,8 @@ RecList reclist; RecList worklist; RecList retrylist; - QMap cardlistmap; + ConflictMap conflictmap; + QMap splistmap; QMap recordidlistmap; QMap titlelistmap; QMap inputgroupmap; @@ -170,6 +187,12 @@ QDateTime livetvTime; int livetvpriority; QMap hasLaterList; + + // cache IsSameProgram() + typedef pair IsSameKey; + typedef QMap IsSameCacheType; + mutable IsSameCacheType cache_is_same_program; + mutable IsSameCacheType cache_is_same_timeslot; }; #endif Index: programs/mythbackend/scheduler.cpp =================================================================== --- programs/mythbackend/scheduler.cpp (revision 14501) +++ programs/mythbackend/scheduler.cpp (working copy) @@ -302,37 +302,37 @@ BuildInputGroupMap(); - VERBOSE(VB_SCHEDULE, "BuildWorkList..."); + VERBOSE(VB_IMPORTANT, "BuildWorkList..."); BuildWorkList(); - VERBOSE(VB_SCHEDULE, "AddNewRecords..."); + VERBOSE(VB_IMPORTANT, "AddNewRecords..."); AddNewRecords(); - VERBOSE(VB_SCHEDULE, "AddNotListed..."); + VERBOSE(VB_IMPORTANT, "AddNotListed..."); AddNotListed(); - VERBOSE(VB_SCHEDULE, "Sort by time..."); + VERBOSE(VB_IMPORTANT, "Sort by time..."); SORT_RECLIST(worklist, comp_overlap); - VERBOSE(VB_SCHEDULE, "PruneOverlaps..."); + VERBOSE(VB_IMPORTANT, "PruneOverlaps..."); PruneOverlaps(); - VERBOSE(VB_SCHEDULE, "Sort by priority..."); + VERBOSE(VB_IMPORTANT, "Sort by priority..."); SORT_RECLIST(worklist, comp_priority); - VERBOSE(VB_SCHEDULE, "BuildListMaps..."); + VERBOSE(VB_IMPORTANT, "BuildListMaps..."); BuildListMaps(); - VERBOSE(VB_SCHEDULE, "SchedNewRecords..."); + VERBOSE(VB_IMPORTANT, "SchedNewRecords..."); SchedNewRecords(); - VERBOSE(VB_SCHEDULE, "SchedPreserveLiveTV..."); + VERBOSE(VB_IMPORTANT, "SchedPreserveLiveTV..."); SchedPreserveLiveTV(); - VERBOSE(VB_SCHEDULE, "ClearListMaps..."); + VERBOSE(VB_IMPORTANT, "ClearListMaps..."); ClearListMaps(); - VERBOSE(VB_SCHEDULE, "Sort by time..."); + VERBOSE(VB_IMPORTANT, "Sort by time..."); SORT_RECLIST(worklist, comp_redundant); - VERBOSE(VB_SCHEDULE, "PruneRedundants..."); + VERBOSE(VB_IMPORTANT, "PruneRedundants..."); PruneRedundants(); - VERBOSE(VB_SCHEDULE, "Sort by time..."); + VERBOSE(VB_IMPORTANT, "Sort by time..."); SORT_RECLIST(worklist, comp_recstart); - VERBOSE(VB_SCHEDULE, "ClearWorkList..."); + VERBOSE(VB_IMPORTANT, "ClearWorkList..."); bool res = ClearWorkList(); return res; @@ -750,8 +750,46 @@ erase_nulls(worklist); } +const static bool is_conflict_dbg = false; +static vector stats_i; +static vector stats_j; +static vector stats_k; +static vector stats_found; +static vector stats_all_i; +static vector stats_all_j; +static vector stats_all_k; +static vector stats_all_found; + +static uint insert(ConflictMap &cmap, const InputGroupList &input_groups, ProgramInfo *p) +{ + uint insert_cnt = 0; + uint64_t btimeX = min(p->recstartts.toTime_t(), + p->startts.toTime_t()); + uint64_t etimeX = max(p->recendts.toTime_t(), + p->endts.toTime_t()); + + InputGroupList::const_iterator it = input_groups.begin(); + for (; it != input_groups.end(); ++it) + { + uint64_t btime = (((uint64_t)(*it)) << 32) + btimeX; + uint64_t etime = (((uint64_t)(*it)) << 32) + etimeX; + for (; btime < etime; btime += 2000) + { + cmap.insert(pair(btime, p)); + insert_cnt++; + } + cmap.insert(pair(etime, p)); + insert_cnt++; + } + + return insert_cnt; +} + void Scheduler::BuildListMaps(void) { + uint tcnt = 0, ccnt = 0; + VERBOSE(VB_IMPORTANT, "BuildListMaps -- begin"); + RecIter i = worklist.begin(); for ( ; i != worklist.end(); i++) { @@ -760,89 +798,190 @@ p->recstatus == rsWillRecord || p->recstatus == rsUnknown) { - cardlistmap[p->cardid].push_back(p); + ++tcnt; + ccnt += insert(conflictmap, inputgroupmap[p->inputid], p); titlelistmap[p->title].push_back(p); recordidlistmap[p->recordid].push_back(p); } } + + VERBOSE(VB_IMPORTANT, "BuildListMaps title cnt: " + <ClearSettingsCache(); + uint max_sz = gContext->GetNumSetting("BuildListMapsTemp", 100); + uint64_t mht = 0, mhc = 0; + + QMap::const_iterator it = titlelistmap.begin(); + for (; it != titlelistmap.end(); ++it) + { + uint sz = (*it).size(); + + if (sz < max_sz) + continue; + + VERBOSE(VB_IMPORTANT, QString("sz %1 title: '%2'") + .arg(sz,4).arg(it.key())); + + uint64_t startt, endt; rdtsc(startt); + for (uint i = 0; i < sz; i++) + splistmap[(*it)[i]] = NULL; + + for (uint i = 0; i < sz; i++) + { + const ProgramInfo *p = (*it)[i]; + for (uint j = i+1; j < sz; j++) + { + const ProgramInfo *q = (*it)[j]; + if (!p->IsSameTimeslot(*q) || !IsSameProgram(p,q)) + continue; + + // place q in p's set + if (!splistmap[p]) + { + splistmap[p] = new RecList; + splistmap[p]->push_back((*it)[i]); + } + + splistmap[q] = splistmap[p]; + splistmap[q]->push_back((*it)[j]); + + break; + } + } + rdtsc(endt); mht += (endt - startt); mhc++; + } + cout<title); - if (!Recording(q)) - continue; - + if (p->cardid != 0 && (p->cardid != q->cardid) && + !GetSharedInputGroup(p->inputid, q->inputid)) + { if (is_conflict_dbg) - cout << QString("\n comparing with '%1' ").arg(q->title); + cout << " cardid== "; + return false; + } - if (p->cardid != 0 && (p->cardid != q->cardid) && - !GetSharedInputGroup(p->inputid, q->inputid)) + if (openEnd && p->chanid != q->chanid) + { + if (p->recendts < q->recstartts || p->recstartts > q->recendts) { if (is_conflict_dbg) - cout << " cardid== "; - continue; + cout << " no-overlap "; + return false; } - - if (openEnd && p->chanid != q->chanid) + } + else + { + if (p->recendts <= q->recstartts || p->recstartts >= q->recendts) { - if (p->recendts < q->recstartts || p->recstartts > q->recendts) - { - if (is_conflict_dbg) - cout << " no-overlap "; - continue; - } + if (is_conflict_dbg) + cout << " no-overlap "; + return false; } - else - { - if (p->recendts <= q->recstartts || p->recstartts >= q->recendts) - { - if (is_conflict_dbg) - cout << " no-overlap "; - continue; - } - } + } - if (is_conflict_dbg) - cout << "\n" << - QString(" cardid's: %1, %2 ").arg(p->cardid).arg(q->cardid) + - QString("Shared input group: %1 ") - .arg(GetSharedInputGroup(p->inputid, q->inputid)) + - QString("mplexid's: %1, %2") - .arg(p->GetMplexID()).arg(q->GetMplexID()); + if (is_conflict_dbg) + { + cout << "\n" << + QString(" cardid's: %1, %2 ").arg(p->cardid).arg(q->cardid) + + QString("Shared input group: %1 ") + .arg(GetSharedInputGroup(p->inputid, q->inputid)) + + QString("mplexid's: %1, %2") + .arg(p->GetMplexID()).arg(q->GetMplexID()); + } - // if two inputs are in the same input group we have a conflict - // unless the programs are on the same multiplex. - if (p->cardid && (p->cardid != q->cardid) && - GetSharedInputGroup(p->inputid, q->inputid) && - p->GetMplexID() && (p->GetMplexID() == q->GetMplexID())) - { - continue; - } + // if two inputs are in the same input group we have a conflict + // unless the programs are on the same multiplex. + if (p->cardid && (p->cardid != q->cardid) && + GetSharedInputGroup(p->inputid, q->inputid) && + p->GetMplexID() && (p->GetMplexID() == q->GetMplexID())) + { + return false; + } - if (is_conflict_dbg) - cout << "\n Found conflict" << endl; + if (is_conflict_dbg) + cout << "\n Found conflict" << endl; - return true; + return true; +} + +bool Scheduler::IsSameProgram( + const ProgramInfo *a, const ProgramInfo *b) const +{ + IsSameKey X(a,b); + IsSameCacheType::const_iterator it = cache_is_same_program.find(X); + if (it != cache_is_same_program.end()) + return *it; + + IsSameKey Y(b,a); + bool val = a->IsSameProgram(*b); + cache_is_same_program[X] = val; + cache_is_same_program[Y] = val; + return val; +} + +bool Scheduler::IsSameTimeslot( + const ProgramInfo *a, const ProgramInfo *b) const +{ + IsSameKey X(a,b); + IsSameCacheType::const_iterator it = cache_is_same_timeslot.find(X); + if (it != cache_is_same_timeslot.end()) + return *it; + + IsSameKey Y(b,a); + bool val = a->IsSameTimeslot(*b); + cache_is_same_timeslot[X] = val; + cache_is_same_timeslot[Y] = val; + return val; +} + +bool Scheduler::FindNextConflict( + const RecList &cardlist, + const ProgramInfo *p, + RecConstIter &j, + bool openEnd) const +{ + cerr<<"FindNextConflict(RecList...)"< &reclists, - const ProgramInfo *p, - bool openend) const +ProgramInfo *Scheduler::FindAnyConflict( + ConflictMap &cmap, const ProgramInfo *p, bool openend) const { - bool is_conflict_dbg = false; + QMap dups; + uint i = 0, j = 0, k = 0; + uint64_t btimeX = p->recstartts.toTime_t() - 2000; + uint64_t etimeX = p->recendts.toTime_t() + 1; - QMap::const_iterator it = reclists.begin(); - for (; it != reclists.end(); ++it) + const InputGroupList &ingrp = inputgroupmap[p->inputid]; + InputGroupList::const_iterator grpit = ingrp.begin(); + const ProgramInfo *last = NULL; + for (; grpit != ingrp.end(); ++grpit) { - if (is_conflict_dbg) + uint64_t btime = btimeX | (((uint64_t)*grpit) << 32); + ConflictMap::iterator beg = cmap.lower_bound(btime); + uint64_t etime = etimeX | (((uint64_t)*grpit) << 32); + ConflictMap::iterator end = cmap.upper_bound(etime); + + ConflictMap::iterator it = beg; + for (; it != end; ++it) { - cout << QString("Checking '%1' for conflicts on cardid %2") - .arg(p->title).arg(it.key()); + if ((it->second == last) || dups[it->second]) + { + i++; + continue; + } + last = dups[it->second] = it->second; + j++; + if (IsConflicting(p, it->second, openend)) + { + stats_i.push_back(i); + stats_j.push_back(j); + stats_k.push_back(k); + stats_found.push_back(1); + return it->second; + } } + } - const RecList &cardlist = *it; - RecConstIter k = cardlist.begin(); - if (FindNextConflict(cardlist, p, k, openend)) + stats_i.push_back(i); + stats_j.push_back(j); + stats_k.push_back(k); + stats_found.push_back(0); + + return NULL; +} + +RecList Scheduler::FindAllConflict( + ConflictMap &cmap, const ProgramInfo *p, bool openend) const +{ + RecList list; + + QMap dups; + uint i = 0, j = 0, k = 0; + uint64_t btimeX = p->recstartts.toTime_t() - 2000; + uint64_t etimeX = p->recendts.toTime_t() + 1; + + const InputGroupList &ingrp = inputgroupmap[p->inputid]; + InputGroupList::const_iterator grpit = ingrp.begin(); + const ProgramInfo *last = NULL; + for (; grpit != ingrp.end(); ++grpit) + { + uint64_t btime = btimeX | (((uint64_t)*grpit) << 32); + ConflictMap::iterator beg = cmap.lower_bound(btime); + uint64_t etime = etimeX | (((uint64_t)*grpit) << 32); + ConflictMap::iterator end = cmap.upper_bound(etime); + + ConflictMap::iterator it = beg; + for (; it != end; ++it) { - return *k; + if ((it->second == last) || dups[it->second]) + { + i++; + continue; + } + last = dups[it->second] = it->second; + + const ProgramInfo *q = last; + if (openend && p->chanid != q->chanid) + { + if (p->recendts < q->recstartts || p->recstartts > q->recendts) + { + k++; + continue; + } + } + else + { + if (p->recendts <= q->recstartts || p->recstartts >= q->recendts) + { + k++; + continue; + } + } + + j++; + if (IsConflicting(p, it->second, openend)) + list.push_back(it->second); } } - return NULL; + stats_all_i.push_back(i); + stats_all_j.push_back(j); + stats_all_k.push_back(k); + stats_all_found.push_back(list.size()); + + return list; } void Scheduler::MarkOtherShowings(ProgramInfo *p) { - RecList *showinglist = &titlelistmap[p->title]; + QMap::iterator sit = splistmap.find(p); + RecList *showinglist = NULL; + if (sit != splistmap.end()) + showinglist = *sit; + else + showinglist = &titlelistmap[p->title]; - MarkShowingsList(*showinglist, p); + if (showinglist) + MarkShowingsList(*showinglist, p); if (p->rectype == kFindOneRecord || p->rectype == kFindDailyRecord || @@ -901,29 +1128,64 @@ void Scheduler::MarkShowingsList(RecList &showinglist, ProgramInfo *p) { + uint me = 0; + uint ign = 0; + uint sametime = 0; + uint later = 0; + uint earlier = 0; + uint other = 0; + RecIter i = showinglist.begin(); for ( ; i != showinglist.end(); i++) { ProgramInfo *q = *i; if (q == p) + { + me++; continue; + } if (q->recstatus != rsUnknown && q->recstatus != rsWillRecord && q->recstatus != rsEarlierShowing && q->recstatus != rsLaterShowing) + { + ign++; continue; + } + if (q->IsSameTimeslot(*p)) + { q->recstatus = rsLaterShowing; + sametime++; + } else if (q->rectype != kSingleRecord && q->rectype != kOverrideRecord && - q->IsSameProgram(*p)) + IsSameProgram(q,p)) { if (q->recstartts < p->recstartts) + { + later++; q->recstatus = rsLaterShowing; + } else + { + earlier++; q->recstatus = rsEarlierShowing; + } } + else + { + other++; + } } + + uint max_sz = gContext->GetNumSetting("BuildListMapsTemp", 100); + if (showinglist.size() >= max_sz) + { + cout<title,20) + <<"m:"<recstatus == rsRecording) return false; - RecList *showinglist = &titlelistmap[p->title]; + QMap::iterator sit = splistmap.find(p); + RecList *showinglist = NULL; + if (sit != splistmap.end()) + showinglist = *sit; + else + showinglist = &titlelistmap[p->title]; + if (!showinglist) + return false; + if (p->rectype == kFindOneRecord || p->rectype == kFindDailyRecord || p->rectype == kFindWeeklyRecord) @@ -980,7 +1250,7 @@ continue; if (!p->IsSameTimeslot(*q)) { - if (!p->IsSameProgram(*q)) + if (!IsSameProgram(p,q)) continue; if ((p->rectype == kSingleRecord || p->rectype == kOverrideRecord)) @@ -1032,7 +1302,7 @@ } } - const ProgramInfo *conflict = FindConflict(cardlistmap, q); + ProgramInfo *conflict = FindAnyConflict(conflictmap, q, false); if (conflict) { PrintRec(conflict, " !"); @@ -1067,21 +1337,38 @@ return false; } +static QMap othershow; + void Scheduler::SchedNewRecords(void) { VERBOSE(VB_SCHEDULE, "Scheduling:"); + uint64_t mot = 0, moc = 0; + uint64_t fat = 0, fac = 0; + uint64_t mot2 = 0, moc2 = 0; + uint64_t mot3 = 0, moc3 = 0; + uint64_t mht = 0, mhc = 0; bool openEnd = (bool)gContext->GetNumSetting("SchedOpenEnd", 0); + uint max_sz = gContext->GetNumSetting("BuildListMapsTemp", 100); + othershow.clear(); + RecIter i = worklist.begin(); while (i != worklist.end()) { ProgramInfo *p = *i; if (p->recstatus == rsRecording) + { + uint64_t startt, endt; rdtsc(startt); MarkOtherShowings(p); + rdtsc(endt); mot += (endt - startt); moc++; + } else if (p->recstatus == rsUnknown) { - const ProgramInfo *conflict = FindConflict(cardlistmap, p, openEnd); + uint64_t startt, endt; rdtsc(startt); + ProgramInfo *conflict = FindAnyConflict(conflictmap, p, openEnd); + rdtsc(endt); fat += (endt - startt); fac++; + if (!conflict) { p->recstatus = rsWillRecord; @@ -1101,7 +1388,23 @@ VERBOSE(VB_GENERAL, msg); } - MarkOtherShowings(p); + RecList *showinglist = &titlelistmap[p->title]; + if (showinglist->size() >= max_sz) + { + othershow[p->title] = showinglist->size(); + uint64_t startt, endt; rdtsc(startt); + MarkOtherShowings(p); + rdtsc(endt); + mot2 += (endt - startt); moc2++; + } + else + { + uint64_t startt, endt; rdtsc(startt); + MarkOtherShowings(p); + rdtsc(endt); + mot3 += (endt - startt); moc3++; + } + PrintRec(p, " +"); } else @@ -1116,14 +1419,43 @@ i++; if (i == worklist.end() || lastpri != (*i)->recpriority) { + uint64_t startt, endt; rdtsc(startt); MoveHigherRecords(); + rdtsc(endt); mht += (endt - startt); mhc++; retrylist.clear(); } } + + cout<=%0:%1 %2").arg(max_sz).arg(moc2,4).arg(mot2,12)<= %1").arg(max_sz)<::iterator it = othershow.begin(); + for (; it != othershow.end(); ++it) + cout<recstatus != rsUnknown) continue; + print_verbose_messages |= VB_SCHEDULE; PrintRec(p, " ?"); + print_verbose_messages &= ~VB_SCHEDULE; + uint64_t startt, endt; rdtsc(startt); + bool cont = false; if (move_this && TryAnotherShowing(p)) + cont = true;; + rdtsc(endt); mht += (endt - startt); mhc++; + if (cont) continue; - BackupRecStatus(); + { + uint64_t startt, endt; rdtsc(startt); + BackupRecStatus(); + rdtsc(endt); bst += (endt - startt); bsc++; + } + p->recstatus = rsWillRecord; if (move_this) + { + uint64_t startt, endt; rdtsc(startt); MarkOtherShowings(p); + rdtsc(endt); mot += (endt - startt); moc++; + } - RecList cardlist; - QMap::const_iterator it = cardlistmap.begin(); - for (; it != cardlistmap.end(); ++it) + + RecList conflicts = FindAllConflict(conflictmap, p, false); + RecList::iterator k = conflicts.begin(); + for (; k != conflicts.end(); ++k) { - RecConstIter it2 = (*it).begin(); - for (; it2 != (*it).end(); ++it2) - cardlist.push_back(*it2); - } - - RecConstIter k = cardlist.begin(); - for ( ; FindNextConflict(cardlist, p, k); k++) - { + uint64_t startt, endt; rdtsc(startt); if ((p->recpriority < (*k)->recpriority && !schedMoveHigher && - move_this) || !TryAnotherShowing(*k, !move_this)) + move_this) || !TryAnotherShowing(*k, !move_this)) { + uint64_t startt, endt; rdtsc(startt); RestoreRecStatus(); + rdtsc(endt); rst += (endt - startt); rsc++; break; } + rdtsc(endt); mht2 += (endt - startt); mhc2++; } if (move_this && p->recstatus == rsWillRecord) PrintRec(p, " +"); } + rdtsc(endt); mht3 += (endt - startt); mhc3++; + + cout<LogEntry("scheduler", LP_INFO, "Scheduled items", msg); + unsigned long long itotal[4] = {0,0,0,0}; + unsigned long long jtotal[4] = {0,0,0,0}; + unsigned long long ktotal[4] = {0,0,0,0}; + unsigned long long cnt[4] = {0,0,0,0}; + uint imax[4] = {0,0,0,0}, jmax[4] = {0,0,0,0}, kmax[4] = {0,0,0,0}; + for (uint i = 0; i < stats_i.size(); i++) + { + uint j = (stats_found[i]) ? 1 : 0; + cnt[j]++; + itotal[j] += stats_i[i]; + imax[j] = max(imax[j], stats_i[i]); + jtotal[j] += stats_j[i]; + jmax[j] = max(jmax[j], stats_j[i]); + ktotal[j] += stats_k[i]; + kmax[j] = max(kmax[j], stats_k[i]); + } + + for (uint i = 0; i < stats_all_i.size(); i++) + { + uint j = 2 + (stats_all_found[i]) ? 1 : 0; + cnt[j] += stats_all_found[i]; + itotal[j] += stats_all_i[i]; + imax[j] = max(imax[j], stats_all_i[i]); + jtotal[j] += stats_all_j[i]; + jmax[j] = max(jmax[j], stats_all_j[i]); + ktotal[j] += stats_all_k[i]; + kmax[j] = max(kmax[j], stats_all_k[i]); + } + + for (uint i = 0; i < 4; i++) + { + VERBOSE(VB_IMPORTANT, + QString("avg i: %1 j: %2, k: %3 max i: %4 j: %5 k: %6") + .arg((double)itotal[i] / (double)cnt[i]) + .arg((double)jtotal[i] / (double)cnt[i]) + .arg((double)ktotal[i] / (double)cnt[i]) + .arg(imax[i]).arg(jmax[i]).arg(kmax[i])); + } + fsInfoCacheFillTime = QDateTime::currentDateTime().addSecs(-1000); lastupdate = curtime;