feat(sync): improve web history parsing and background TMDB enrichment
- Rewrite Netflix parser to extract viewing history via the Shakti API using performance resource entries. - Update Prime Video URLs and replace Disney+/Apple TV with a generic "Other" provider. - Add batched database transactions for runtime and poster updates to prevent UI notification storms. - Decouple TMDB enrichment into a background coroutine scope so large imports return immediately. - Track loaded provider in the sync screen to properly trigger WebView reloads on provider switches.
This commit is contained in:
parent
f55ff411c6
commit
73529d4c7e
SSH key fingerprint:
SHA256:2pKKXDZucWvaF/GzXNz0FY53EAO1YDLN80bqS+TTz/o
5 changed files with 562 additions and 139 deletions
|
|
@ -4,9 +4,19 @@ import androidx.room.Dao
|
|||
import androidx.room.Insert
|
||||
import androidx.room.OnConflictStrategy
|
||||
import androidx.room.Query
|
||||
import androidx.room.Transaction
|
||||
import com.bingestats.app.data.model.ViewingItem
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
|
||||
/**
|
||||
* A batch of runtime/poster updates to apply in a single DB transaction.
|
||||
*/
|
||||
data class RuntimeUpdate(
|
||||
val showTitle: String,
|
||||
val durationMinutes: Int,
|
||||
val posterPath: String?
|
||||
)
|
||||
|
||||
data class ShowAggregation(
|
||||
val showTitle: String,
|
||||
val provider: String,
|
||||
|
|
@ -105,6 +115,17 @@ interface ViewingDao {
|
|||
@Query("UPDATE viewing_items SET durationMinutes = :duration, posterPath = :posterPath WHERE showTitle = :showTitle")
|
||||
suspend fun updateShowRuntimeAndPoster(showTitle: String, duration: Int, posterPath: String?)
|
||||
|
||||
/**
|
||||
* Apply many runtime/poster updates in a single transaction so Room emits only one
|
||||
* change notification (instead of one per title, which would storm the stats Flows).
|
||||
*/
|
||||
@Transaction
|
||||
suspend fun updateRuntimesBatch(updates: List<RuntimeUpdate>) {
|
||||
updates.forEach { u ->
|
||||
updateShowRuntimeAndPoster(u.showTitle, u.durationMinutes, u.posterPath)
|
||||
}
|
||||
}
|
||||
|
||||
@Query("DELETE FROM viewing_items")
|
||||
suspend fun clearAll()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,22 +24,15 @@ enum class StreamingProvider(
|
|||
id = "prime",
|
||||
displayName = "Prime Video",
|
||||
brandColorHex = "#00A8E1",
|
||||
defaultWebUrl = "https://www.amazon.com/ap/signin",
|
||||
viewingHistoryUrl = "https://www.amazon.com/gp/your-account/order-history"
|
||||
defaultWebUrl = "https://www.primevideo.com",
|
||||
viewingHistoryUrl = "https://www.primevideo.com/settings/watch-history"
|
||||
),
|
||||
DISNEY_PLUS(
|
||||
id = "disney",
|
||||
displayName = "Disney+",
|
||||
brandColorHex = "#113CCF",
|
||||
defaultWebUrl = "https://www.disneyplus.com/login",
|
||||
viewingHistoryUrl = "https://www.disneyplus.com/account"
|
||||
),
|
||||
APPLE_TV(
|
||||
id = "apple_tv",
|
||||
displayName = "Apple TV+",
|
||||
brandColorHex = "#A2A2A2",
|
||||
defaultWebUrl = "https://tv.apple.com",
|
||||
viewingHistoryUrl = "https://tv.apple.com/settings"
|
||||
OTHER(
|
||||
id = "other",
|
||||
displayName = "Andere",
|
||||
brandColorHex = "#A0A0A0",
|
||||
defaultWebUrl = "",
|
||||
viewingHistoryUrl = ""
|
||||
);
|
||||
|
||||
val primaryColor: Color
|
||||
|
|
@ -47,7 +40,7 @@ enum class StreamingProvider(
|
|||
|
||||
companion object {
|
||||
fun fromId(id: String): StreamingProvider {
|
||||
return entries.find { it.id.equals(id, ignoreCase = true) } ?: NETFLIX
|
||||
return entries.find { it.id.equals(id, ignoreCase = true) } ?: OTHER
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,76 +15,288 @@ package com.bingestats.app.data.parser
|
|||
object WebSyncScripts {
|
||||
|
||||
/**
|
||||
* Netflix: read BUILD_IDENTIFIER + authURL from the page's global context, then
|
||||
* page through /api/shakti/<build>/viewingactivity collecting viewedItems.
|
||||
* Returns a JSON array string (or "[]" on failure).
|
||||
* Netflix: extract the full viewing history via the internal Shakti API.
|
||||
*
|
||||
* The page itself already fetched the viewing activity while loading, so the exact
|
||||
* API URL (including all required query params like authURL) is present in the
|
||||
* browser's performance resource entries. We clone that URL and only swap the page
|
||||
* number for pagination. Building the URL ourselves (even with a correct authURL)
|
||||
* yields HTTP 421 "Misdirected Request", because Netflix rejects hand-crafted
|
||||
* requests via HTTP/2 connection coalescing checks.
|
||||
*
|
||||
* Fallbacks: reactContext BUILD_IDENTIFIER, HTML regex, and finally a DOM scrape.
|
||||
* Each step logs to console (logcat tag BingeStats_Web).
|
||||
* Returns a Promise resolving to a JSON array string of viewing items.
|
||||
*/
|
||||
val netflix: String = """
|
||||
(function() {
|
||||
try {
|
||||
var ctx = window.netflix && window.netflix.reactContext;
|
||||
if (!ctx) return JSON.stringify([]);
|
||||
var serverDefs = ctx.models && ctx.models.serverDefs && ctx.models.serverDefs.data;
|
||||
if (!serverDefs || !serverDefs.BUILD_IDENTIFIER) return JSON.stringify([]);
|
||||
var buildId = serverDefs.BUILD_IDENTIFIER;
|
||||
var authURL = "";
|
||||
function log(msg) { try { console.log('[BingeStats] ' + msg); } catch(e){} }
|
||||
|
||||
// Find the real viewingactivity URL the page itself used (preferred) or
|
||||
// reconstruct a root from reactContext/HTML as a last resort.
|
||||
function discoverApi() {
|
||||
// --- Preferred: performance resource entries (the SPA just called it) ---
|
||||
try {
|
||||
authURL = ctx.models.memberContext.data.userInfo.data.authURL
|
||||
|| ctx.models.userInfo.data.authURL || "";
|
||||
} catch (e) {}
|
||||
var entries = performance.getEntriesByType('resource').map(function(e){ return e.name; });
|
||||
for (var i = 0; i < entries.length; i++) {
|
||||
if (entries[i].indexOf('viewingactivity') !== -1) {
|
||||
log('perf viewingactivity URL gefunden');
|
||||
return { mode: 'url', template: entries[i] };
|
||||
}
|
||||
}
|
||||
log('kein viewingactivity perf-Eintrag (' + entries.length + ' resources)');
|
||||
} catch (e) { log('perf Fehler: ' + e.message); }
|
||||
|
||||
var collected = [];
|
||||
var pageSize = 100;
|
||||
var pg = 0;
|
||||
var total = -1;
|
||||
// --- Fallback A: reactContext build id (+ authURL) ---
|
||||
try {
|
||||
var n = window.netflix;
|
||||
if (n) {
|
||||
var rc = n.reactContext || n.appContext;
|
||||
if (rc && rc.models) {
|
||||
var sd = rc.models.serverDefs && rc.models.serverDefs.data;
|
||||
if (sd && sd.BUILD_IDENTIFIER) {
|
||||
var auth = "";
|
||||
try {
|
||||
var mc = rc.models.memberContext || rc.models.userInfo;
|
||||
auth = (mc && mc.data && (mc.data.userInfo && mc.data.userInfo.data && mc.data.userInfo.data.authURL))
|
||||
|| (mc && mc.data && mc.data.authURL) || "";
|
||||
} catch (e) {}
|
||||
log('reactContext OK, build=' + sd.BUILD_IDENTIFIER + (auth ? ' +auth' : ' ohne auth'));
|
||||
return { mode: 'build', root: '/api/shakti/' + sd.BUILD_IDENTIFIER, auth: auth };
|
||||
}
|
||||
}
|
||||
}
|
||||
log('reactContext nicht gefunden');
|
||||
} catch (e) { log('reactContext Fehler: ' + e.message); }
|
||||
|
||||
// --- Fallback B: regex shakti path out of the page HTML ---
|
||||
try {
|
||||
var html = document.documentElement.outerHTML;
|
||||
var m = html.match(/\/api\/shakti\/([a-zA-Z0-9]+)\//);
|
||||
if (m) {
|
||||
log('HTML regex build=' + m[1]);
|
||||
return { mode: 'build', root: '/api/shakti/' + m[1], auth: '' };
|
||||
}
|
||||
log('kein shakti-Pfad im HTML');
|
||||
} catch (e) { log('HTML regex Fehler: ' + e.message); }
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
// Build the fetch URL for a given page number from the discovered API info.
|
||||
function urlForPage(api, n) {
|
||||
if (api.mode === 'url') {
|
||||
// Clone the captured URL, replace/insert pg and pgsize params.
|
||||
var u = api.template;
|
||||
u = u.replace(/([?&])pg=[^&]*/, '$1pg=' + n);
|
||||
u = u.replace(/([?&])pgsize=[^&]*/, '$1pgsize=100');
|
||||
if (u.indexOf('pg=') === -1) u += (u.indexOf('?') === -1 ? '?' : '&') + 'pg=' + n;
|
||||
if (u.indexOf('pgsize=') === -1) u += '&pgsize=100';
|
||||
return u;
|
||||
}
|
||||
var u = api.root + '/viewingactivity?pg=' + n + '&pgsize=100';
|
||||
if (api.auth) u += '&authURL=' + encodeURIComponent(api.auth);
|
||||
return u;
|
||||
}
|
||||
|
||||
return new Promise(function(resolve) {
|
||||
try {
|
||||
// DOM scraper: Netflix's viewing-activity page is server-rendered, so the
|
||||
// rows are present in the DOM. The exact class names are hashed/unstable,
|
||||
// so we try several selector strategies and pick whichever yields items.
|
||||
// Read one snapshot of the currently rendered rows.
|
||||
function scrapeDomOnce() {
|
||||
var out = [];
|
||||
|
||||
// Strategy A: <li> rows whose text looks like "Title ... Date".
|
||||
var lis = document.querySelectorAll('li');
|
||||
for (var i = 0; i < lis.length; i++) {
|
||||
try {
|
||||
var li = lis[i];
|
||||
var txt = li.textContent.replace(/\s+/g, ' ').trim();
|
||||
if (txt.length < 3 || txt.length > 300) continue;
|
||||
// Split trailing date token (e.g. "Show Name 01.02.24" or "... 12/31/23").
|
||||
var m = txt.match(/^(.*?)[\s]+(\d{1,2}[\.\/]\d{1,2}[\.\/]\d{2,4})$/);
|
||||
if (m) {
|
||||
out.push({ key: txt, title: m[1].trim(), date: m[2] });
|
||||
} else {
|
||||
out.push({ key: txt, title: txt, date: null });
|
||||
}
|
||||
} catch (e) {}
|
||||
}
|
||||
if (out.length) return out;
|
||||
|
||||
// Strategy B: any element carrying a data-title or aria-label.
|
||||
var labelled = document.querySelectorAll('[data-title], [aria-label]');
|
||||
for (var j = 0; j < labelled.length; j++) {
|
||||
try {
|
||||
var t = (labelled[j].getAttribute('data-title') || labelled[j].getAttribute('aria-label') || '').trim();
|
||||
if (t.length < 3 || t.length > 300) continue;
|
||||
out.push({ key: t, title: t, date: null });
|
||||
} catch (e) {}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Netflix lazy-loads older entries as you scroll (infinite scroll). So we
|
||||
// scroll to the bottom repeatedly, snapshotting new rows into a deduped map
|
||||
// until the count stops growing, then resolve with everything collected.
|
||||
function scrollAndCollect(onDone) {
|
||||
var seen = {};
|
||||
var collected = [];
|
||||
var stableRounds = 0;
|
||||
var lastCount = -1;
|
||||
var attempt = 0;
|
||||
|
||||
// Seed with the currently visible rows (logged once for debugging).
|
||||
var firstBatch = document.querySelectorAll('li');
|
||||
if (firstBatch.length) {
|
||||
log('DOM: ' + firstBatch.length + ' <li> Elemente, erstes: ' + firstBatch[0].outerHTML.substring(0, 160));
|
||||
}
|
||||
|
||||
// Netflix lazy-loads older entries via a "Show more" / "Mehr anzeigen"
|
||||
// BUTTON (not infinite scroll), so we click it after each snapshot.
|
||||
function clickShowMore() {
|
||||
var btns = document.querySelectorAll('button, a');
|
||||
for (var i = 0; i < btns.length; i++) {
|
||||
var b = btns[i];
|
||||
var txt = (b.textContent || '').trim().toLowerCase();
|
||||
if (txt.length < 2 || txt.length > 40) continue;
|
||||
if (txt.indexOf('mehr anzeigen') !== -1 ||
|
||||
txt.indexOf('show more') !== -1 ||
|
||||
txt.indexOf('ver más') !== -1 ||
|
||||
txt.indexOf('afficher plus') !== -1 ||
|
||||
txt === 'mehr' || txt === 'more') {
|
||||
b.click();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function tick() {
|
||||
attempt++;
|
||||
var snap = scrapeDomOnce();
|
||||
var added = 0;
|
||||
for (var i = 0; i < snap.length; i++) {
|
||||
var it = snap[i];
|
||||
if (!seen[it.key]) {
|
||||
seen[it.key] = true;
|
||||
collected.push({ title: it.title, date: it.date });
|
||||
added++;
|
||||
}
|
||||
}
|
||||
if (collected.length === lastCount) {
|
||||
stableRounds++;
|
||||
} else {
|
||||
stableRounds = 0;
|
||||
lastCount = collected.length;
|
||||
}
|
||||
log('Schritt ' + attempt + ': ' + added + ' neu, gesamt ' + collected.length + ' (stable ' + stableRounds + ')');
|
||||
// Stop after the list stops growing (button gone), or at a hard cap.
|
||||
if (stableRounds >= 4 || attempt > 500 || collected.length > 6000) {
|
||||
onDone(collected);
|
||||
} else {
|
||||
var clicked = clickShowMore();
|
||||
if (!clicked) window.scrollTo(0, document.body.scrollHeight);
|
||||
setTimeout(tick, 800);
|
||||
}
|
||||
}
|
||||
tick();
|
||||
}
|
||||
|
||||
var api = discoverApi();
|
||||
// If the page is server-rendered (no perf entry) the hand-built API call
|
||||
// returns HTTP 421. Prefer the DOM scrape right away in that case: it is
|
||||
// more reliable than fighting Netflix's anti-bot on the API.
|
||||
if (!api || api.mode !== 'url') {
|
||||
log('Server-gerendert oder keine URL -> DOM-Scrape mit Auto-Scroll');
|
||||
scrollAndCollect(function(items) {
|
||||
log('DOM-Scrape fertig: ' + items.length + ' Eintraege');
|
||||
resolve(JSON.stringify(items));
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
var collected = [];
|
||||
var pg = 0;
|
||||
var total = -1;
|
||||
|
||||
// Synchronous loop over pages. fetch() is async, so we chain via a
|
||||
// recursive helper and resolve the outer Promise.
|
||||
return new Promise(function(resolve) {
|
||||
function loadPage(n) {
|
||||
var url = '/api/shakti/' + buildId + '/viewingactivity?pg=' + n + '&pgsize=' + pageSize;
|
||||
if (authURL) url += '&authURL=' + encodeURIComponent(authURL);
|
||||
fetch(url, { credentials: 'include' })
|
||||
.then(function(r) { return r.ok ? r.json() : null; })
|
||||
.then(function(data) {
|
||||
if (!data) { resolve(JSON.stringify(collected)); return; }
|
||||
var url = urlForPage(api, n);
|
||||
log('lade Seite ' + n + ': ' + url.substring(0, 120));
|
||||
fetch(url, { credentials: 'include', headers: { 'Accept': 'application/json' } })
|
||||
.then(function(r) {
|
||||
log('Seite ' + n + ' HTTP ' + r.status);
|
||||
return r.ok ? r.text() : null;
|
||||
})
|
||||
.then(function(text) {
|
||||
if (!text) {
|
||||
// If the very first page fails, fall back to the scrolling
|
||||
// DOM scrape before giving up.
|
||||
if (n === 0) {
|
||||
log('API-Fehler -> DOM-Scrape mit Auto-Scroll');
|
||||
scrollAndCollect(function(items) {
|
||||
log('DOM-Scrape fertig: ' + items.length + ' Eintraege');
|
||||
resolve(JSON.stringify(items));
|
||||
});
|
||||
} else {
|
||||
resolve(JSON.stringify(collected));
|
||||
}
|
||||
return;
|
||||
}
|
||||
var data;
|
||||
try { data = JSON.parse(text); } catch (e) {
|
||||
log('JSON-Parse-Fehler Seite ' + n + ': ' + text.substring(0, 120));
|
||||
resolve(JSON.stringify(collected)); return;
|
||||
}
|
||||
var items = data.viewedItems || [];
|
||||
collected = collected.concat(items);
|
||||
if (total < 0) total = data.vhSize || 0;
|
||||
pg++;
|
||||
// Stop when a page is empty or we've read everything.
|
||||
log('Seite ' + n + ': ' + items.length + ' Items, gesamt ' + collected.length + '/' + total);
|
||||
if (items.length === 0 || (total > 0 && collected.length >= total) || pg > 200) {
|
||||
resolve(JSON.stringify(collected));
|
||||
} else {
|
||||
loadPage(pg);
|
||||
}
|
||||
})
|
||||
.catch(function() { resolve(JSON.stringify(collected)); });
|
||||
.catch(function(err) {
|
||||
log('fetch Fehler Seite ' + n + ': ' + err.message);
|
||||
resolve(JSON.stringify(collected));
|
||||
});
|
||||
}
|
||||
loadPage(0);
|
||||
});
|
||||
} catch (e) {
|
||||
return JSON.stringify([]);
|
||||
}
|
||||
} catch (e) {
|
||||
log('Fehler: ' + e.message);
|
||||
resolve(JSON.stringify([]));
|
||||
}
|
||||
});
|
||||
})()
|
||||
""".trimIndent()
|
||||
|
||||
|
||||
/**
|
||||
* Prime Video: parse the inline JSON embedded in <script type="text/template"> tags.
|
||||
* The structure is props -> widgets[] -> content.content.titles[] -> date sections,
|
||||
* each section has .titles[] with {title:{text}, time, children:[...] }.
|
||||
* Returns a JSON array string of {title, episodeTitle, time, duration}.
|
||||
* Prime Video: extract the full watch history.
|
||||
*
|
||||
* Prime's /settings/watch-history page loads the first batch as inline JSON inside
|
||||
* <script type="text/template"> tags and lazy-loads older entries via fetch() as you
|
||||
* scroll. So this script does three things:
|
||||
* 1. Monkey-patches window.fetch so every lazy-loaded response is captured.
|
||||
* 2. Parses both the inline JSON and each captured response with the same widget
|
||||
* walker (props -> widgets -> content.content.titles -> date sections -> titles).
|
||||
* 3. Auto-scrolls to the bottom until the page stops growing, then resolves.
|
||||
*
|
||||
* Returns a Promise that resolves to a JSON array of
|
||||
* {title|seriesTitle+episodeTitle, time, duration}.
|
||||
*/
|
||||
val prime: String = """
|
||||
(function() {
|
||||
try {
|
||||
var results = [];
|
||||
var scripts = document.querySelectorAll('script[type="text/template"]');
|
||||
for (var i = 0; i < scripts.length; i++) {
|
||||
var raw = scripts[i].textContent.trim();
|
||||
if (!raw || raw.charAt(0) !== '{') continue;
|
||||
var obj;
|
||||
try { obj = JSON.parse(raw); } catch (e) { continue; }
|
||||
var collected = {};
|
||||
|
||||
// Walker: given a parsed object, pull every watch-history item out of it.
|
||||
function harvest(obj) {
|
||||
var widgets = (obj && obj.props) ? obj.props : [];
|
||||
if (!Array.isArray(widgets)) widgets = [widgets];
|
||||
widgets.forEach(function(w) {
|
||||
|
|
@ -95,26 +307,77 @@ object WebSyncScripts {
|
|||
var items = section.titles || [];
|
||||
items.forEach(function(t) {
|
||||
if (!t || !t.title || !t.title.text) return;
|
||||
results.push({
|
||||
var key = (t.gti || t.title.text) + '|' + (t.time || '');
|
||||
if (collected[key]) return;
|
||||
collected[key] = {
|
||||
title: t.title.text,
|
||||
time: t.time || null,
|
||||
duration: t.duration || null
|
||||
});
|
||||
};
|
||||
(t.children || []).forEach(function(c) {
|
||||
if (!c || !c.title || !c.title.text) return;
|
||||
results.push({
|
||||
var ckey = (c.gti || c.title.text) + '|' + (c.time || '');
|
||||
if (collected[ckey]) return;
|
||||
collected[ckey] = {
|
||||
seriesTitle: t.title.text,
|
||||
episodeTitle: c.title.text,
|
||||
time: c.time || t.time || null,
|
||||
duration: c.duration || null
|
||||
});
|
||||
};
|
||||
});
|
||||
});
|
||||
});
|
||||
} catch (e) {}
|
||||
});
|
||||
}
|
||||
return JSON.stringify(results);
|
||||
|
||||
// 1. Parse the initial inline JSON.
|
||||
var scripts = document.querySelectorAll('script[type="text/template"]');
|
||||
for (var i = 0; i < scripts.length; i++) {
|
||||
var raw = scripts[i].textContent.trim();
|
||||
if (!raw || raw.charAt(0) !== '{') continue;
|
||||
try { harvest(JSON.parse(raw)); } catch (e) {}
|
||||
}
|
||||
|
||||
// 2. Intercept lazy-loaded fetch responses.
|
||||
if (!window.__bingePrimeFetchPatched) {
|
||||
window.__bingePrimeFetchPatched = true;
|
||||
var origFetch = window.fetch;
|
||||
window.fetch = function() {
|
||||
var p = origFetch.apply(this, arguments);
|
||||
return p.then(function(resp) {
|
||||
var clone = resp.clone();
|
||||
clone.json().then(function(data) {
|
||||
try { harvest(data); } catch (e) {}
|
||||
}).catch(function() {});
|
||||
return resp;
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
// 3. Auto-scroll until the page stops growing, then resolve.
|
||||
return new Promise(function(resolve) {
|
||||
var lastCount = -1, stableRounds = 0, attempt = 0;
|
||||
function tick() {
|
||||
attempt++;
|
||||
var count = Object.keys(collected).length;
|
||||
window.scrollTo(0, document.body.scrollHeight);
|
||||
if (count === lastCount) {
|
||||
stableRounds++;
|
||||
} else {
|
||||
stableRounds = 0;
|
||||
lastCount = count;
|
||||
}
|
||||
// Stop after 3 stable rounds in a row, or hard cap on attempts.
|
||||
if (stableRounds >= 3 || attempt > 60) {
|
||||
var list = Object.keys(collected).map(function(k) { return collected[k]; });
|
||||
resolve(JSON.stringify(list));
|
||||
} else {
|
||||
setTimeout(tick, 700);
|
||||
}
|
||||
}
|
||||
tick();
|
||||
});
|
||||
} catch (e) {
|
||||
return JSON.stringify([]);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package com.bingestats.app.data.repository
|
|||
import com.bingestats.app.data.db.MovieAggregation
|
||||
import com.bingestats.app.data.db.MonthlyAggregation
|
||||
import com.bingestats.app.data.db.ProviderAggregation
|
||||
import com.bingestats.app.data.db.RuntimeUpdate
|
||||
import com.bingestats.app.data.db.ShowAggregation
|
||||
import com.bingestats.app.data.db.ViewingDatabase
|
||||
import com.bingestats.app.data.model.DayOfWeekStat
|
||||
|
|
@ -15,14 +16,21 @@ import com.bingestats.app.data.model.TmdbCacheEntity
|
|||
import com.bingestats.app.data.model.ViewingItem
|
||||
import com.bingestats.app.data.model.WatchStats
|
||||
import com.bingestats.app.data.remote.TmdbClient
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.flow.Flow
|
||||
import kotlinx.coroutines.flow.combine
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.withContext
|
||||
import java.util.Calendar
|
||||
|
||||
class StatsRepository(private val db: ViewingDatabase) {
|
||||
|
||||
// Long-lived scope for background TMDB enrichment, decoupled from the importing
|
||||
// coroutine so large imports return immediately while posters/runtimes stream in.
|
||||
private val enrichmentScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
|
||||
|
||||
private val viewingDao = db.viewingDao()
|
||||
private val tmdbCacheDao = db.tmdbCacheDao()
|
||||
private val tmdbClient = TmdbClient()
|
||||
|
|
@ -111,30 +119,34 @@ class StatsRepository(private val db: ViewingDatabase) {
|
|||
}
|
||||
|
||||
suspend fun importItems(items: List<ViewingItem>) = withContext(Dispatchers.IO) {
|
||||
// Insert fast (batched) so the UI updates immediately. TMDB enrichment is slow
|
||||
// (hundreds of sequential API calls for large histories), so we run it decoupled
|
||||
// in the background — posters/runtimes stream in afterwards.
|
||||
viewingDao.insertAll(items)
|
||||
enrichWithTmdbData(items)
|
||||
enrichmentScope.launch { enrichWithTmdbData(items) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Asynchronously query TMDB API for exact show runtimes and poster art.
|
||||
* Query TMDB for exact show runtimes and poster art, then apply the resulting DB
|
||||
* updates in batched transactions so the stats Flows don't emit hundreds of times.
|
||||
*/
|
||||
private suspend fun enrichWithTmdbData(items: List<ViewingItem>) = withContext(Dispatchers.IO) {
|
||||
val uniqueShows = items.map { it.showTitle to it.contentType }.distinctBy { it.first }
|
||||
val pendingUpdates = mutableListOf<RuntimeUpdate>()
|
||||
|
||||
for ((showTitle, contentType) in uniqueShows) {
|
||||
val cacheKey = "${contentType.name.lowercase()}_${showTitle.lowercase()}"
|
||||
val existingCache = tmdbCacheDao.getCache(cacheKey)
|
||||
|
||||
if (existingCache != null) {
|
||||
if (existingCache.runtimeMinutes > 0) {
|
||||
viewingDao.updateShowRuntimeAndPoster(
|
||||
showTitle = showTitle,
|
||||
duration = existingCache.runtimeMinutes,
|
||||
posterPath = existingCache.posterPath
|
||||
)
|
||||
}
|
||||
val runtime: Int
|
||||
val posterPath: String?
|
||||
if (existingCache != null && existingCache.runtimeMinutes > 0) {
|
||||
runtime = existingCache.runtimeMinutes
|
||||
posterPath = existingCache.posterPath
|
||||
} else {
|
||||
val (runtime, posterPath) = tmdbClient.searchAndGetRuntime(showTitle, contentType)
|
||||
val (rt, pp) = tmdbClient.searchAndGetRuntime(showTitle, contentType)
|
||||
runtime = rt
|
||||
posterPath = pp
|
||||
tmdbCacheDao.insertCache(
|
||||
TmdbCacheEntity(
|
||||
titleKey = cacheKey,
|
||||
|
|
@ -145,12 +157,18 @@ class StatsRepository(private val db: ViewingDatabase) {
|
|||
posterPath = posterPath
|
||||
)
|
||||
)
|
||||
viewingDao.updateShowRuntimeAndPoster(
|
||||
showTitle = showTitle,
|
||||
duration = runtime,
|
||||
posterPath = posterPath
|
||||
)
|
||||
}
|
||||
pendingUpdates.add(RuntimeUpdate(showTitle, runtime, posterPath))
|
||||
|
||||
// Flush in chunks so the UI gets a single notification per chunk instead of
|
||||
// hundreds of per-title notifications that would freeze navigation.
|
||||
if (pendingUpdates.size >= 50) {
|
||||
viewingDao.updateRuntimesBatch(pendingUpdates.toList())
|
||||
pendingUpdates.clear()
|
||||
}
|
||||
}
|
||||
if (pendingUpdates.isNotEmpty()) {
|
||||
viewingDao.updateRuntimesBatch(pendingUpdates.toList())
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,8 +2,10 @@ package com.bingestats.app.ui.screens
|
|||
|
||||
import android.annotation.SuppressLint
|
||||
import android.net.Uri
|
||||
import android.util.Log
|
||||
import android.webkit.CookieManager
|
||||
import android.webkit.WebResourceRequest
|
||||
import android.webkit.WebChromeClient
|
||||
import android.webkit.WebView
|
||||
import android.webkit.WebViewClient
|
||||
import androidx.activity.compose.BackHandler
|
||||
|
|
@ -242,6 +244,8 @@ fun WebViewSyncSection(
|
|||
var isExtracting by remember { mutableStateOf(false) }
|
||||
var statusMessage by remember { mutableStateOf<String?>(null) }
|
||||
val webViewRef = remember { mutableStateOf<WebView?>(null) }
|
||||
// Track which provider the WebView currently shows so a provider switch reloads it.
|
||||
var loadedProvider by remember { mutableStateOf<StreamingProvider?>(null) }
|
||||
|
||||
// Clean up the WebView when the composable leaves the composition. Without this the
|
||||
// WebView can keep running JS/holding the screen, which previously blocked navigation.
|
||||
|
|
@ -284,8 +288,15 @@ fun WebViewSyncSection(
|
|||
onClick = {
|
||||
val webview = webViewRef.value ?: return@Button
|
||||
isExtracting = true
|
||||
statusMessage = "Lese Verlauf aus…"
|
||||
statusMessage = if (provider == StreamingProvider.NETFLIX) {
|
||||
"Löse CSV-Download aus…"
|
||||
} else {
|
||||
"Rufe Verlauf ab…"
|
||||
}
|
||||
runExtraction(webview, provider) { result ->
|
||||
// Only reached for Prime (scrape). Netflix is handled by the
|
||||
// DownloadListener, which calls onExtractHistory directly.
|
||||
android.util.Log.d("BingeStats_Web", "[Kotlin] onResult: length=${result?.length}, value=${result?.take(200)}")
|
||||
isExtracting = false
|
||||
statusMessage = null
|
||||
if (!result.isNullOrBlank() && result != "[]" && result != "null") {
|
||||
|
|
@ -294,6 +305,12 @@ fun WebViewSyncSection(
|
|||
statusMessage = "Keine Daten gefunden. Stelle sicher, dass du eingeloggt bist und sich der Verlauf geladen hat."
|
||||
}
|
||||
}
|
||||
// Netflix: the script only clicks the download button; the result
|
||||
// arrives via the DownloadListener, so stop the spinner shortly after
|
||||
// unless a download has started.
|
||||
if (provider == StreamingProvider.NETFLIX) {
|
||||
statusMessage = "CSV-Download ausgelöst. Falls nichts passiert, lade die Seite neu."
|
||||
}
|
||||
},
|
||||
colors = ButtonDefaults.buttonColors(
|
||||
containerColor = provider.primaryColor,
|
||||
|
|
@ -331,108 +348,219 @@ fun WebViewSyncSection(
|
|||
return
|
||||
}
|
||||
|
||||
Box(modifier = Modifier.fillMaxSize()) {
|
||||
AndroidView(
|
||||
factory = { context ->
|
||||
WebView(context).apply {
|
||||
settings.javaScriptEnabled = true
|
||||
settings.domStorageEnabled = true
|
||||
// Allow third-party cookies so login flows survive the redirect chain.
|
||||
CookieManager.getInstance().setAcceptThirdPartyCookies(this, true)
|
||||
settings.userAgentString = "Mozilla/5.0 (Linux; Android 10; Mobile) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36"
|
||||
Box(modifier = Modifier.fillMaxSize()) {
|
||||
AndroidView(
|
||||
factory = { context ->
|
||||
WebView(context).apply {
|
||||
settings.javaScriptEnabled = true
|
||||
settings.domStorageEnabled = true
|
||||
// Allow third-party cookies so login flows survive the redirect chain.
|
||||
CookieManager.getInstance().setAcceptThirdPartyCookies(this, true)
|
||||
settings.userAgentString = "Mozilla/5.0 (Linux; Android 10; Mobile) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36"
|
||||
|
||||
webViewClient = object : WebViewClient() {
|
||||
override fun onPageFinished(view: WebView?, url: String?) {
|
||||
super.onPageFinished(view, url)
|
||||
currentUrl = url ?: ""
|
||||
isLoadingWeb = false
|
||||
// Install the JS<->Kotlin bridge: the extraction promise resolves
|
||||
// into window.__bingeResult, which runExtraction() then polls.
|
||||
view?.evaluateJavascript(
|
||||
"window.__bingeResult = null; window.__bingeExtracted = function(v) { window.__bingeResult = (typeof v === 'string') ? v : JSON.stringify(v); };",
|
||||
null
|
||||
)
|
||||
}
|
||||
|
||||
override fun shouldOverrideUrlLoading(view: WebView?, request: WebResourceRequest?): Boolean {
|
||||
currentUrl = request?.url.toString()
|
||||
return false
|
||||
}
|
||||
webViewClient = object : WebViewClient() {
|
||||
override fun onPageFinished(view: WebView?, url: String?) {
|
||||
super.onPageFinished(view, url)
|
||||
currentUrl = url ?: ""
|
||||
isLoadingWeb = false
|
||||
}
|
||||
loadUrl(provider.viewingHistoryUrl)
|
||||
webViewRef.value = this
|
||||
}
|
||||
},
|
||||
// Removed the old aggressive reload: only react to provider changes, not every recomposition.
|
||||
update = { _ -> },
|
||||
modifier = Modifier
|
||||
.fillMaxSize()
|
||||
.border(1.dp, GlassCardBorder, RoundedCornerShape(8.dp))
|
||||
)
|
||||
|
||||
if (isLoadingWeb || isExtracting) {
|
||||
CircularProgressIndicator(
|
||||
color = provider.primaryColor,
|
||||
modifier = Modifier.align(Alignment.Center)
|
||||
)
|
||||
}
|
||||
override fun shouldOverrideUrlLoading(view: WebView?, request: WebResourceRequest?): Boolean {
|
||||
currentUrl = request?.url.toString()
|
||||
return false
|
||||
}
|
||||
}
|
||||
loadUrl(provider.viewingHistoryUrl)
|
||||
// Forward JS console.log to logcat so extraction can be debugged live.
|
||||
webChromeClient = object : WebChromeClient() {
|
||||
override fun onConsoleMessage(consoleMessage: android.webkit.ConsoleMessage): Boolean {
|
||||
Log.d("BingeStats_Web", "[${consoleMessage.messageLevel()}] ${consoleMessage.message()}")
|
||||
return true
|
||||
}
|
||||
}
|
||||
loadedProvider = provider
|
||||
webViewRef.value = this
|
||||
}
|
||||
},
|
||||
// Reload only when the provider actually changes — not on every recomposition.
|
||||
update = { webView ->
|
||||
if (loadedProvider != provider) {
|
||||
isLoadingWeb = true
|
||||
webView.loadUrl(provider.viewingHistoryUrl)
|
||||
loadedProvider = provider
|
||||
}
|
||||
},
|
||||
modifier = Modifier
|
||||
.fillMaxSize()
|
||||
.border(1.dp, GlassCardBorder, RoundedCornerShape(8.dp))
|
||||
)
|
||||
|
||||
if (isLoadingWeb || isExtracting) {
|
||||
CircularProgressIndicator(
|
||||
color = provider.primaryColor,
|
||||
modifier = Modifier.align(Alignment.Center)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Injects the provider-specific extraction script into the WebView. The Netflix script
|
||||
* returns a Promise, so we wrap evaluation in a `.then()` and read the resolved JSON.
|
||||
* Injects the provider-specific extraction script into the WebView. The scripts return a
|
||||
* Promise; we attach a then()-handler that stores the resolved value, then poll the
|
||||
* stored value until it appears (async fetch/scroll takes time).
|
||||
*
|
||||
* Bridge encoding: evaluateJavascript returns any string value JSON-encoded (i.e. wrapped
|
||||
* in quotes and escaped). The stored __bingeResult is already a JSON string from the
|
||||
* script, so when read back it arrives double-encoded. We unwrap exactly one layer.
|
||||
*/
|
||||
private fun runExtraction(
|
||||
webView: WebView,
|
||||
provider: StreamingProvider,
|
||||
onResult: (String?) -> Unit
|
||||
) {
|
||||
// Netflix: Netflix generates the CSV entirely client-side as a Blob, so there is no
|
||||
// HTTP URL we can fetch with OkHttp. Instead we intercept the Blob right where it is
|
||||
// created: we wrap document.createElement('a') so that when Netflix triggers the
|
||||
// download, we read the Blob text via FileReader and store it. Then we click the
|
||||
// "Download all" button and poll for the captured CSV text.
|
||||
if (provider == StreamingProvider.NETFLIX) {
|
||||
val setupAndClick = """
|
||||
try {
|
||||
function log(m){ try{ console.log('[BingeStats] '+m); }catch(e){} }
|
||||
window.__bingeDone = false;
|
||||
window.__bingeResult = null;
|
||||
// Intercept the anchor-click that triggers the blob download. Netflix builds
|
||||
// an <a download href="blob:..."> and clicks it. We wrap it to read the Blob.
|
||||
if (!window.__bingeBlobPatched) {
|
||||
window.__bingeBlobPatched = true;
|
||||
var origCreate = document.createElement.bind(document);
|
||||
document.createElement = function(tag) {
|
||||
var el = origCreate(tag);
|
||||
if (String(tag).toLowerCase() === 'a') {
|
||||
var origClick = el.click.bind(el);
|
||||
el.click = function() {
|
||||
var href = el.href || el.getAttribute('href') || '';
|
||||
if (href.indexOf('blob:') === 0 && el.href) {
|
||||
try {
|
||||
fetch(href).then(function(r){ return r.text(); }).then(function(txt){
|
||||
log('Blob abgefangen: ' + txt.length + ' bytes');
|
||||
window.__bingeResult = txt;
|
||||
window.__bingeDone = true;
|
||||
}).catch(function(e){ log('Blob-Lesefehler: ' + e.message); });
|
||||
} catch (e) { log('Blob-Fetch Fehler: ' + e.message); }
|
||||
}
|
||||
return origClick();
|
||||
};
|
||||
}
|
||||
return el;
|
||||
};
|
||||
}
|
||||
// Find and click the download button.
|
||||
var btns = document.querySelectorAll('button, a, [role="button"]');
|
||||
var hit = null;
|
||||
for (var i = 0; i < btns.length; i++) {
|
||||
var t = (btns[i].textContent || '').trim().toLowerCase();
|
||||
if (t.indexOf('alle herunterladen') !== -1 ||
|
||||
t.indexOf('download all') !== -1 ||
|
||||
t.indexOf('alle exportieren') !== -1 ||
|
||||
t === 'export' || t === 'exportieren') {
|
||||
hit = btns[i]; break;
|
||||
}
|
||||
}
|
||||
if (hit) { log('Klicke Download-Button: ' + hit.textContent.trim()); hit.click(); }
|
||||
else {
|
||||
log('Kein Download-Button gefunden. Verfuegbare Buttons:');
|
||||
document.querySelectorAll('button, a').forEach(function(b){
|
||||
var tx=(b.textContent||'').trim(); if(tx.length>0&&tx.length<40) log(' -> "'+tx+'"');
|
||||
});
|
||||
}
|
||||
} catch (e) { console.log('[BingeStats] Fehler: ' + e.message); }
|
||||
""".trimIndent()
|
||||
webView.evaluateJavascript(setupAndClick, null)
|
||||
// The blob interception resolves asynchronously; poll for the captured CSV text.
|
||||
pollExtraction(webView, attempts = 0, onResult = onResult)
|
||||
return
|
||||
}
|
||||
|
||||
val script = when (provider) {
|
||||
StreamingProvider.NETFLIX -> WebSyncScripts.netflix
|
||||
StreamingProvider.AMAZON_PRIME -> WebSyncScripts.prime
|
||||
else -> { onResult(null); return }
|
||||
}
|
||||
|
||||
// The script string is itself an IIFE "(function(){...})()" that returns either a value
|
||||
// or a Promise. We capture its return value directly and store the resolved/strungified
|
||||
// result into window.__bingeResult, which pollExtraction then reads.
|
||||
val wrapped = """
|
||||
(function() {
|
||||
var r = (function() { $script })();
|
||||
try {
|
||||
var r = $script;
|
||||
window.__bingeResult = null;
|
||||
window.__bingeDone = false;
|
||||
if (r && typeof r.then === 'function') {
|
||||
r.then(function(v) { __bingeExtracted(v); }).catch(function() { __bingeExtracted('[]'); });
|
||||
r.then(function(v) {
|
||||
window.__bingeResult = (typeof v === 'string') ? v : JSON.stringify(v);
|
||||
window.__bingeDone = true;
|
||||
}).catch(function(e) {
|
||||
window.__bingeResult = '[]';
|
||||
window.__bingeDone = true;
|
||||
});
|
||||
} else {
|
||||
__bingeExtracted(r);
|
||||
window.__bingeResult = (typeof r === 'string') ? r : JSON.stringify(r);
|
||||
window.__bingeDone = true;
|
||||
}
|
||||
})();
|
||||
} catch (e) {
|
||||
window.__bingeResult = '[]';
|
||||
window.__bingeDone = true;
|
||||
}
|
||||
""".trimIndent()
|
||||
|
||||
// Bridge from JS back to Kotlin: store the result in a JS variable we then retrieve.
|
||||
webView.evaluateJavascript(wrapped, null)
|
||||
// Small delay so the async fetch (Netflix pagination) has time to complete before polling.
|
||||
pollExtraction(webView, attempts = 0, onResult = onResult)
|
||||
}
|
||||
|
||||
private fun pollExtraction(webView: WebView, attempts: Int, onResult: (String?) -> Unit) {
|
||||
// The injected promise calls __bingeExtracted(v); we expose it via addJavascriptInterface-like
|
||||
// retrieval: evaluate a getter. Cap attempts so we never spin forever.
|
||||
val maxAttempts = 20
|
||||
// Clicking through thousands of "Show more" pages takes a while, so allow generous
|
||||
// polling time before giving up.
|
||||
val maxAttempts = 300
|
||||
if (attempts > maxAttempts) {
|
||||
onResult(null)
|
||||
return
|
||||
}
|
||||
webView.postDelayed({
|
||||
webView.evaluateJavascript(
|
||||
"(window.__bingeResult === undefined) ? null : window.__bingeResult;"
|
||||
"window.__bingeDone ? window.__bingeResult : null;"
|
||||
) { raw ->
|
||||
if (raw == null || raw == "null" || raw == "undefined") {
|
||||
android.util.Log.d("BingeStats_Web", "[Kotlin] poll attempt=$attempts raw=${raw?.take(120)}")
|
||||
// evaluateJavascript returns the value JSON-encoded: a string arrives as
|
||||
// "\"...\"" and null as "null". Unwrap one JSON string layer.
|
||||
val unwrapped = unwrapJsString(raw)
|
||||
if (unwrapped == null) {
|
||||
pollExtraction(webView, attempts + 1, onResult)
|
||||
} else {
|
||||
onResult(raw)
|
||||
onResult(unwrapped)
|
||||
}
|
||||
}
|
||||
}, 500)
|
||||
}
|
||||
|
||||
/**
|
||||
* evaluateJavascript returns JSON-encoded values. A JS string "abc" comes back as
|
||||
* "\"abc\"". null/undefined come back as "null". This unwraps the outer JSON quotes for
|
||||
* a string and returns null for actual null/undefined.
|
||||
*/
|
||||
private fun unwrapJsString(raw: String?): String? {
|
||||
if (raw == null || raw == "null" || raw == "undefined") return null
|
||||
val trimmed = raw.trim()
|
||||
// Already a JSON string literal -> unwrap one quoting layer.
|
||||
if (trimmed.startsWith("\"") && trimmed.endsWith("\"")) {
|
||||
return trimmed.substring(1, trimmed.length - 1)
|
||||
.replace("\\\"", "\"")
|
||||
.replace("\\\\", "\\")
|
||||
.replace("\\n", "\n")
|
||||
}
|
||||
return trimmed
|
||||
}
|
||||
|
||||
@Composable
|
||||
fun CsvImportSection(
|
||||
provider: StreamingProvider,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue