feat(sync): improve web history parsing and background TMDB enrichment

- Rewrite Netflix parser to extract viewing history via the Shakti API using performance resource entries.
- Update Prime Video URLs and replace Disney+/Apple TV with a generic "Other" provider.
- Add batched database transactions for runtime and poster updates to prevent UI notification storms.
- Decouple TMDB enrichment into a background coroutine scope so large imports return immediately.
- Track loaded provider in the sync screen to properly trigger WebView reloads on provider switches.
This commit is contained in:
Tronax 2026-08-04 19:39:00 +02:00
parent f55ff411c6
commit 73529d4c7e
Signed by: Tronax
SSH key fingerprint: SHA256:2pKKXDZucWvaF/GzXNz0FY53EAO1YDLN80bqS+TTz/o
5 changed files with 562 additions and 139 deletions

View file

@ -4,9 +4,19 @@ import androidx.room.Dao
import androidx.room.Insert import androidx.room.Insert
import androidx.room.OnConflictStrategy import androidx.room.OnConflictStrategy
import androidx.room.Query import androidx.room.Query
import androidx.room.Transaction
import com.bingestats.app.data.model.ViewingItem import com.bingestats.app.data.model.ViewingItem
import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.Flow
/**
* A batch of runtime/poster updates to apply in a single DB transaction.
*/
data class RuntimeUpdate(
val showTitle: String,
val durationMinutes: Int,
val posterPath: String?
)
data class ShowAggregation( data class ShowAggregation(
val showTitle: String, val showTitle: String,
val provider: String, val provider: String,
@ -105,6 +115,17 @@ interface ViewingDao {
@Query("UPDATE viewing_items SET durationMinutes = :duration, posterPath = :posterPath WHERE showTitle = :showTitle") @Query("UPDATE viewing_items SET durationMinutes = :duration, posterPath = :posterPath WHERE showTitle = :showTitle")
suspend fun updateShowRuntimeAndPoster(showTitle: String, duration: Int, posterPath: String?) suspend fun updateShowRuntimeAndPoster(showTitle: String, duration: Int, posterPath: String?)
/**
* Apply many runtime/poster updates in a single transaction so Room emits only one
* change notification (instead of one per title, which would storm the stats Flows).
*/
@Transaction
suspend fun updateRuntimesBatch(updates: List<RuntimeUpdate>) {
updates.forEach { u ->
updateShowRuntimeAndPoster(u.showTitle, u.durationMinutes, u.posterPath)
}
}
@Query("DELETE FROM viewing_items") @Query("DELETE FROM viewing_items")
suspend fun clearAll() suspend fun clearAll()
} }

View file

@ -24,22 +24,15 @@ enum class StreamingProvider(
id = "prime", id = "prime",
displayName = "Prime Video", displayName = "Prime Video",
brandColorHex = "#00A8E1", brandColorHex = "#00A8E1",
defaultWebUrl = "https://www.amazon.com/ap/signin", defaultWebUrl = "https://www.primevideo.com",
viewingHistoryUrl = "https://www.amazon.com/gp/your-account/order-history" viewingHistoryUrl = "https://www.primevideo.com/settings/watch-history"
), ),
DISNEY_PLUS( OTHER(
id = "disney", id = "other",
displayName = "Disney+", displayName = "Andere",
brandColorHex = "#113CCF", brandColorHex = "#A0A0A0",
defaultWebUrl = "https://www.disneyplus.com/login", defaultWebUrl = "",
viewingHistoryUrl = "https://www.disneyplus.com/account" viewingHistoryUrl = ""
),
APPLE_TV(
id = "apple_tv",
displayName = "Apple TV+",
brandColorHex = "#A2A2A2",
defaultWebUrl = "https://tv.apple.com",
viewingHistoryUrl = "https://tv.apple.com/settings"
); );
val primaryColor: Color val primaryColor: Color
@ -47,7 +40,7 @@ enum class StreamingProvider(
companion object { companion object {
fun fromId(id: String): StreamingProvider { fun fromId(id: String): StreamingProvider {
return entries.find { it.id.equals(id, ignoreCase = true) } ?: NETFLIX return entries.find { it.id.equals(id, ignoreCase = true) } ?: OTHER
} }
} }
} }

View file

@ -15,76 +15,288 @@ package com.bingestats.app.data.parser
object WebSyncScripts { object WebSyncScripts {
/** /**
* Netflix: read BUILD_IDENTIFIER + authURL from the page's global context, then * Netflix: extract the full viewing history via the internal Shakti API.
* page through /api/shakti/<build>/viewingactivity collecting viewedItems. *
* Returns a JSON array string (or "[]" on failure). * The page itself already fetched the viewing activity while loading, so the exact
* API URL (including all required query params like authURL) is present in the
* browser's performance resource entries. We clone that URL and only swap the page
* number for pagination. Building the URL ourselves (even with a correct authURL)
* yields HTTP 421 "Misdirected Request", because Netflix rejects hand-crafted
* requests via HTTP/2 connection coalescing checks.
*
* Fallbacks: reactContext BUILD_IDENTIFIER, HTML regex, and finally a DOM scrape.
* Each step logs to console (logcat tag BingeStats_Web).
* Returns a Promise resolving to a JSON array string of viewing items.
*/ */
val netflix: String = """ val netflix: String = """
(function() { (function() {
function log(msg) { try { console.log('[BingeStats] ' + msg); } catch(e){} }
// Find the real viewingactivity URL the page itself used (preferred) or
// reconstruct a root from reactContext/HTML as a last resort.
function discoverApi() {
// --- Preferred: performance resource entries (the SPA just called it) ---
try { try {
var ctx = window.netflix && window.netflix.reactContext; var entries = performance.getEntriesByType('resource').map(function(e){ return e.name; });
if (!ctx) return JSON.stringify([]); for (var i = 0; i < entries.length; i++) {
var serverDefs = ctx.models && ctx.models.serverDefs && ctx.models.serverDefs.data; if (entries[i].indexOf('viewingactivity') !== -1) {
if (!serverDefs || !serverDefs.BUILD_IDENTIFIER) return JSON.stringify([]); log('perf viewingactivity URL gefunden');
var buildId = serverDefs.BUILD_IDENTIFIER; return { mode: 'url', template: entries[i] };
var authURL = ""; }
}
log('kein viewingactivity perf-Eintrag (' + entries.length + ' resources)');
} catch (e) { log('perf Fehler: ' + e.message); }
// --- Fallback A: reactContext build id (+ authURL) ---
try { try {
authURL = ctx.models.memberContext.data.userInfo.data.authURL var n = window.netflix;
|| ctx.models.userInfo.data.authURL || ""; if (n) {
var rc = n.reactContext || n.appContext;
if (rc && rc.models) {
var sd = rc.models.serverDefs && rc.models.serverDefs.data;
if (sd && sd.BUILD_IDENTIFIER) {
var auth = "";
try {
var mc = rc.models.memberContext || rc.models.userInfo;
auth = (mc && mc.data && (mc.data.userInfo && mc.data.userInfo.data && mc.data.userInfo.data.authURL))
|| (mc && mc.data && mc.data.authURL) || "";
} catch (e) {} } catch (e) {}
log('reactContext OK, build=' + sd.BUILD_IDENTIFIER + (auth ? ' +auth' : ' ohne auth'));
return { mode: 'build', root: '/api/shakti/' + sd.BUILD_IDENTIFIER, auth: auth };
}
}
}
log('reactContext nicht gefunden');
} catch (e) { log('reactContext Fehler: ' + e.message); }
// --- Fallback B: regex shakti path out of the page HTML ---
try {
var html = document.documentElement.outerHTML;
var m = html.match(/\/api\/shakti\/([a-zA-Z0-9]+)\//);
if (m) {
log('HTML regex build=' + m[1]);
return { mode: 'build', root: '/api/shakti/' + m[1], auth: '' };
}
log('kein shakti-Pfad im HTML');
} catch (e) { log('HTML regex Fehler: ' + e.message); }
return null;
}
// Build the fetch URL for a given page number from the discovered API info.
function urlForPage(api, n) {
if (api.mode === 'url') {
// Clone the captured URL, replace/insert pg and pgsize params.
var u = api.template;
u = u.replace(/([?&])pg=[^&]*/, '$1pg=' + n);
u = u.replace(/([?&])pgsize=[^&]*/, '$1pgsize=100');
if (u.indexOf('pg=') === -1) u += (u.indexOf('?') === -1 ? '?' : '&') + 'pg=' + n;
if (u.indexOf('pgsize=') === -1) u += '&pgsize=100';
return u;
}
var u = api.root + '/viewingactivity?pg=' + n + '&pgsize=100';
if (api.auth) u += '&authURL=' + encodeURIComponent(api.auth);
return u;
}
return new Promise(function(resolve) {
try {
// DOM scraper: Netflix's viewing-activity page is server-rendered, so the
// rows are present in the DOM. The exact class names are hashed/unstable,
// so we try several selector strategies and pick whichever yields items.
// Read one snapshot of the currently rendered rows.
function scrapeDomOnce() {
var out = [];
// Strategy A: <li> rows whose text looks like "Title ... Date".
var lis = document.querySelectorAll('li');
for (var i = 0; i < lis.length; i++) {
try {
var li = lis[i];
var txt = li.textContent.replace(/\s+/g, ' ').trim();
if (txt.length < 3 || txt.length > 300) continue;
// Split trailing date token (e.g. "Show Name 01.02.24" or "... 12/31/23").
var m = txt.match(/^(.*?)[\s]+(\d{1,2}[\.\/]\d{1,2}[\.\/]\d{2,4})$/);
if (m) {
out.push({ key: txt, title: m[1].trim(), date: m[2] });
} else {
out.push({ key: txt, title: txt, date: null });
}
} catch (e) {}
}
if (out.length) return out;
// Strategy B: any element carrying a data-title or aria-label.
var labelled = document.querySelectorAll('[data-title], [aria-label]');
for (var j = 0; j < labelled.length; j++) {
try {
var t = (labelled[j].getAttribute('data-title') || labelled[j].getAttribute('aria-label') || '').trim();
if (t.length < 3 || t.length > 300) continue;
out.push({ key: t, title: t, date: null });
} catch (e) {}
}
return out;
}
// Netflix lazy-loads older entries as you scroll (infinite scroll). So we
// scroll to the bottom repeatedly, snapshotting new rows into a deduped map
// until the count stops growing, then resolve with everything collected.
function scrollAndCollect(onDone) {
var seen = {};
var collected = [];
var stableRounds = 0;
var lastCount = -1;
var attempt = 0;
// Seed with the currently visible rows (logged once for debugging).
var firstBatch = document.querySelectorAll('li');
if (firstBatch.length) {
log('DOM: ' + firstBatch.length + ' <li> Elemente, erstes: ' + firstBatch[0].outerHTML.substring(0, 160));
}
// Netflix lazy-loads older entries via a "Show more" / "Mehr anzeigen"
// BUTTON (not infinite scroll), so we click it after each snapshot.
function clickShowMore() {
var btns = document.querySelectorAll('button, a');
for (var i = 0; i < btns.length; i++) {
var b = btns[i];
var txt = (b.textContent || '').trim().toLowerCase();
if (txt.length < 2 || txt.length > 40) continue;
if (txt.indexOf('mehr anzeigen') !== -1 ||
txt.indexOf('show more') !== -1 ||
txt.indexOf('ver más') !== -1 ||
txt.indexOf('afficher plus') !== -1 ||
txt === 'mehr' || txt === 'more') {
b.click();
return true;
}
}
return false;
}
function tick() {
attempt++;
var snap = scrapeDomOnce();
var added = 0;
for (var i = 0; i < snap.length; i++) {
var it = snap[i];
if (!seen[it.key]) {
seen[it.key] = true;
collected.push({ title: it.title, date: it.date });
added++;
}
}
if (collected.length === lastCount) {
stableRounds++;
} else {
stableRounds = 0;
lastCount = collected.length;
}
log('Schritt ' + attempt + ': ' + added + ' neu, gesamt ' + collected.length + ' (stable ' + stableRounds + ')');
// Stop after the list stops growing (button gone), or at a hard cap.
if (stableRounds >= 4 || attempt > 500 || collected.length > 6000) {
onDone(collected);
} else {
var clicked = clickShowMore();
if (!clicked) window.scrollTo(0, document.body.scrollHeight);
setTimeout(tick, 800);
}
}
tick();
}
var api = discoverApi();
// If the page is server-rendered (no perf entry) the hand-built API call
// returns HTTP 421. Prefer the DOM scrape right away in that case: it is
// more reliable than fighting Netflix's anti-bot on the API.
if (!api || api.mode !== 'url') {
log('Server-gerendert oder keine URL -> DOM-Scrape mit Auto-Scroll');
scrollAndCollect(function(items) {
log('DOM-Scrape fertig: ' + items.length + ' Eintraege');
resolve(JSON.stringify(items));
});
return;
}
var collected = []; var collected = [];
var pageSize = 100;
var pg = 0; var pg = 0;
var total = -1; var total = -1;
// Synchronous loop over pages. fetch() is async, so we chain via a
// recursive helper and resolve the outer Promise.
return new Promise(function(resolve) {
function loadPage(n) { function loadPage(n) {
var url = '/api/shakti/' + buildId + '/viewingactivity?pg=' + n + '&pgsize=' + pageSize; var url = urlForPage(api, n);
if (authURL) url += '&authURL=' + encodeURIComponent(authURL); log('lade Seite ' + n + ': ' + url.substring(0, 120));
fetch(url, { credentials: 'include' }) fetch(url, { credentials: 'include', headers: { 'Accept': 'application/json' } })
.then(function(r) { return r.ok ? r.json() : null; }) .then(function(r) {
.then(function(data) { log('Seite ' + n + ' HTTP ' + r.status);
if (!data) { resolve(JSON.stringify(collected)); return; } return r.ok ? r.text() : null;
})
.then(function(text) {
if (!text) {
// If the very first page fails, fall back to the scrolling
// DOM scrape before giving up.
if (n === 0) {
log('API-Fehler -> DOM-Scrape mit Auto-Scroll');
scrollAndCollect(function(items) {
log('DOM-Scrape fertig: ' + items.length + ' Eintraege');
resolve(JSON.stringify(items));
});
} else {
resolve(JSON.stringify(collected));
}
return;
}
var data;
try { data = JSON.parse(text); } catch (e) {
log('JSON-Parse-Fehler Seite ' + n + ': ' + text.substring(0, 120));
resolve(JSON.stringify(collected)); return;
}
var items = data.viewedItems || []; var items = data.viewedItems || [];
collected = collected.concat(items); collected = collected.concat(items);
if (total < 0) total = data.vhSize || 0; if (total < 0) total = data.vhSize || 0;
pg++; pg++;
// Stop when a page is empty or we've read everything. log('Seite ' + n + ': ' + items.length + ' Items, gesamt ' + collected.length + '/' + total);
if (items.length === 0 || (total > 0 && collected.length >= total) || pg > 200) { if (items.length === 0 || (total > 0 && collected.length >= total) || pg > 200) {
resolve(JSON.stringify(collected)); resolve(JSON.stringify(collected));
} else { } else {
loadPage(pg); loadPage(pg);
} }
}) })
.catch(function() { resolve(JSON.stringify(collected)); }); .catch(function(err) {
log('fetch Fehler Seite ' + n + ': ' + err.message);
resolve(JSON.stringify(collected));
});
} }
loadPage(0); loadPage(0);
});
} catch (e) { } catch (e) {
return JSON.stringify([]); log('Fehler: ' + e.message);
resolve(JSON.stringify([]));
} }
});
})() })()
""".trimIndent() """.trimIndent()
/** /**
* Prime Video: parse the inline JSON embedded in <script type="text/template"> tags. * Prime Video: extract the full watch history.
* The structure is props -> widgets[] -> content.content.titles[] -> date sections, *
* each section has .titles[] with {title:{text}, time, children:[...] }. * Prime's /settings/watch-history page loads the first batch as inline JSON inside
* Returns a JSON array string of {title, episodeTitle, time, duration}. * <script type="text/template"> tags and lazy-loads older entries via fetch() as you
* scroll. So this script does three things:
* 1. Monkey-patches window.fetch so every lazy-loaded response is captured.
* 2. Parses both the inline JSON and each captured response with the same widget
* walker (props -> widgets -> content.content.titles -> date sections -> titles).
* 3. Auto-scrolls to the bottom until the page stops growing, then resolves.
*
* Returns a Promise that resolves to a JSON array of
* {title|seriesTitle+episodeTitle, time, duration}.
*/ */
val prime: String = """ val prime: String = """
(function() { (function() {
try { try {
var results = []; var collected = {};
var scripts = document.querySelectorAll('script[type="text/template"]');
for (var i = 0; i < scripts.length; i++) { // Walker: given a parsed object, pull every watch-history item out of it.
var raw = scripts[i].textContent.trim(); function harvest(obj) {
if (!raw || raw.charAt(0) !== '{') continue;
var obj;
try { obj = JSON.parse(raw); } catch (e) { continue; }
var widgets = (obj && obj.props) ? obj.props : []; var widgets = (obj && obj.props) ? obj.props : [];
if (!Array.isArray(widgets)) widgets = [widgets]; if (!Array.isArray(widgets)) widgets = [widgets];
widgets.forEach(function(w) { widgets.forEach(function(w) {
@ -95,26 +307,77 @@ object WebSyncScripts {
var items = section.titles || []; var items = section.titles || [];
items.forEach(function(t) { items.forEach(function(t) {
if (!t || !t.title || !t.title.text) return; if (!t || !t.title || !t.title.text) return;
results.push({ var key = (t.gti || t.title.text) + '|' + (t.time || '');
if (collected[key]) return;
collected[key] = {
title: t.title.text, title: t.title.text,
time: t.time || null, time: t.time || null,
duration: t.duration || null duration: t.duration || null
}); };
(t.children || []).forEach(function(c) { (t.children || []).forEach(function(c) {
if (!c || !c.title || !c.title.text) return; if (!c || !c.title || !c.title.text) return;
results.push({ var ckey = (c.gti || c.title.text) + '|' + (c.time || '');
if (collected[ckey]) return;
collected[ckey] = {
seriesTitle: t.title.text, seriesTitle: t.title.text,
episodeTitle: c.title.text, episodeTitle: c.title.text,
time: c.time || t.time || null, time: c.time || t.time || null,
duration: c.duration || null duration: c.duration || null
}); };
}); });
}); });
}); });
} catch (e) {} } catch (e) {}
}); });
} }
return JSON.stringify(results);
// 1. Parse the initial inline JSON.
var scripts = document.querySelectorAll('script[type="text/template"]');
for (var i = 0; i < scripts.length; i++) {
var raw = scripts[i].textContent.trim();
if (!raw || raw.charAt(0) !== '{') continue;
try { harvest(JSON.parse(raw)); } catch (e) {}
}
// 2. Intercept lazy-loaded fetch responses.
if (!window.__bingePrimeFetchPatched) {
window.__bingePrimeFetchPatched = true;
var origFetch = window.fetch;
window.fetch = function() {
var p = origFetch.apply(this, arguments);
return p.then(function(resp) {
var clone = resp.clone();
clone.json().then(function(data) {
try { harvest(data); } catch (e) {}
}).catch(function() {});
return resp;
});
};
}
// 3. Auto-scroll until the page stops growing, then resolve.
return new Promise(function(resolve) {
var lastCount = -1, stableRounds = 0, attempt = 0;
function tick() {
attempt++;
var count = Object.keys(collected).length;
window.scrollTo(0, document.body.scrollHeight);
if (count === lastCount) {
stableRounds++;
} else {
stableRounds = 0;
lastCount = count;
}
// Stop after 3 stable rounds in a row, or hard cap on attempts.
if (stableRounds >= 3 || attempt > 60) {
var list = Object.keys(collected).map(function(k) { return collected[k]; });
resolve(JSON.stringify(list));
} else {
setTimeout(tick, 700);
}
}
tick();
});
} catch (e) { } catch (e) {
return JSON.stringify([]); return JSON.stringify([]);
} }

View file

@ -3,6 +3,7 @@ package com.bingestats.app.data.repository
import com.bingestats.app.data.db.MovieAggregation import com.bingestats.app.data.db.MovieAggregation
import com.bingestats.app.data.db.MonthlyAggregation import com.bingestats.app.data.db.MonthlyAggregation
import com.bingestats.app.data.db.ProviderAggregation import com.bingestats.app.data.db.ProviderAggregation
import com.bingestats.app.data.db.RuntimeUpdate
import com.bingestats.app.data.db.ShowAggregation import com.bingestats.app.data.db.ShowAggregation
import com.bingestats.app.data.db.ViewingDatabase import com.bingestats.app.data.db.ViewingDatabase
import com.bingestats.app.data.model.DayOfWeekStat import com.bingestats.app.data.model.DayOfWeekStat
@ -15,14 +16,21 @@ import com.bingestats.app.data.model.TmdbCacheEntity
import com.bingestats.app.data.model.ViewingItem import com.bingestats.app.data.model.ViewingItem
import com.bingestats.app.data.model.WatchStats import com.bingestats.app.data.model.WatchStats
import com.bingestats.app.data.remote.TmdbClient import com.bingestats.app.data.remote.TmdbClient
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.combine import kotlinx.coroutines.flow.combine
import kotlinx.coroutines.launch
import kotlinx.coroutines.withContext import kotlinx.coroutines.withContext
import java.util.Calendar import java.util.Calendar
class StatsRepository(private val db: ViewingDatabase) { class StatsRepository(private val db: ViewingDatabase) {
// Long-lived scope for background TMDB enrichment, decoupled from the importing
// coroutine so large imports return immediately while posters/runtimes stream in.
private val enrichmentScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val viewingDao = db.viewingDao() private val viewingDao = db.viewingDao()
private val tmdbCacheDao = db.tmdbCacheDao() private val tmdbCacheDao = db.tmdbCacheDao()
private val tmdbClient = TmdbClient() private val tmdbClient = TmdbClient()
@ -111,30 +119,34 @@ class StatsRepository(private val db: ViewingDatabase) {
} }
suspend fun importItems(items: List<ViewingItem>) = withContext(Dispatchers.IO) { suspend fun importItems(items: List<ViewingItem>) = withContext(Dispatchers.IO) {
// Insert fast (batched) so the UI updates immediately. TMDB enrichment is slow
// (hundreds of sequential API calls for large histories), so we run it decoupled
// in the background — posters/runtimes stream in afterwards.
viewingDao.insertAll(items) viewingDao.insertAll(items)
enrichWithTmdbData(items) enrichmentScope.launch { enrichWithTmdbData(items) }
} }
/** /**
* Asynchronously query TMDB API for exact show runtimes and poster art. * Query TMDB for exact show runtimes and poster art, then apply the resulting DB
* updates in batched transactions so the stats Flows don't emit hundreds of times.
*/ */
private suspend fun enrichWithTmdbData(items: List<ViewingItem>) = withContext(Dispatchers.IO) { private suspend fun enrichWithTmdbData(items: List<ViewingItem>) = withContext(Dispatchers.IO) {
val uniqueShows = items.map { it.showTitle to it.contentType }.distinctBy { it.first } val uniqueShows = items.map { it.showTitle to it.contentType }.distinctBy { it.first }
val pendingUpdates = mutableListOf<RuntimeUpdate>()
for ((showTitle, contentType) in uniqueShows) { for ((showTitle, contentType) in uniqueShows) {
val cacheKey = "${contentType.name.lowercase()}_${showTitle.lowercase()}" val cacheKey = "${contentType.name.lowercase()}_${showTitle.lowercase()}"
val existingCache = tmdbCacheDao.getCache(cacheKey) val existingCache = tmdbCacheDao.getCache(cacheKey)
if (existingCache != null) { val runtime: Int
if (existingCache.runtimeMinutes > 0) { val posterPath: String?
viewingDao.updateShowRuntimeAndPoster( if (existingCache != null && existingCache.runtimeMinutes > 0) {
showTitle = showTitle, runtime = existingCache.runtimeMinutes
duration = existingCache.runtimeMinutes,
posterPath = existingCache.posterPath posterPath = existingCache.posterPath
)
}
} else { } else {
val (runtime, posterPath) = tmdbClient.searchAndGetRuntime(showTitle, contentType) val (rt, pp) = tmdbClient.searchAndGetRuntime(showTitle, contentType)
runtime = rt
posterPath = pp
tmdbCacheDao.insertCache( tmdbCacheDao.insertCache(
TmdbCacheEntity( TmdbCacheEntity(
titleKey = cacheKey, titleKey = cacheKey,
@ -145,12 +157,18 @@ class StatsRepository(private val db: ViewingDatabase) {
posterPath = posterPath posterPath = posterPath
) )
) )
viewingDao.updateShowRuntimeAndPoster(
showTitle = showTitle,
duration = runtime,
posterPath = posterPath
)
} }
pendingUpdates.add(RuntimeUpdate(showTitle, runtime, posterPath))
// Flush in chunks so the UI gets a single notification per chunk instead of
// hundreds of per-title notifications that would freeze navigation.
if (pendingUpdates.size >= 50) {
viewingDao.updateRuntimesBatch(pendingUpdates.toList())
pendingUpdates.clear()
}
}
if (pendingUpdates.isNotEmpty()) {
viewingDao.updateRuntimesBatch(pendingUpdates.toList())
} }
} }

View file

@ -2,8 +2,10 @@ package com.bingestats.app.ui.screens
import android.annotation.SuppressLint import android.annotation.SuppressLint
import android.net.Uri import android.net.Uri
import android.util.Log
import android.webkit.CookieManager import android.webkit.CookieManager
import android.webkit.WebResourceRequest import android.webkit.WebResourceRequest
import android.webkit.WebChromeClient
import android.webkit.WebView import android.webkit.WebView
import android.webkit.WebViewClient import android.webkit.WebViewClient
import androidx.activity.compose.BackHandler import androidx.activity.compose.BackHandler
@ -242,6 +244,8 @@ fun WebViewSyncSection(
var isExtracting by remember { mutableStateOf(false) } var isExtracting by remember { mutableStateOf(false) }
var statusMessage by remember { mutableStateOf<String?>(null) } var statusMessage by remember { mutableStateOf<String?>(null) }
val webViewRef = remember { mutableStateOf<WebView?>(null) } val webViewRef = remember { mutableStateOf<WebView?>(null) }
// Track which provider the WebView currently shows so a provider switch reloads it.
var loadedProvider by remember { mutableStateOf<StreamingProvider?>(null) }
// Clean up the WebView when the composable leaves the composition. Without this the // Clean up the WebView when the composable leaves the composition. Without this the
// WebView can keep running JS/holding the screen, which previously blocked navigation. // WebView can keep running JS/holding the screen, which previously blocked navigation.
@ -284,8 +288,15 @@ fun WebViewSyncSection(
onClick = { onClick = {
val webview = webViewRef.value ?: return@Button val webview = webViewRef.value ?: return@Button
isExtracting = true isExtracting = true
statusMessage = "Lese Verlauf aus…" statusMessage = if (provider == StreamingProvider.NETFLIX) {
"Löse CSV-Download aus…"
} else {
"Rufe Verlauf ab…"
}
runExtraction(webview, provider) { result -> runExtraction(webview, provider) { result ->
// Only reached for Prime (scrape). Netflix is handled by the
// DownloadListener, which calls onExtractHistory directly.
android.util.Log.d("BingeStats_Web", "[Kotlin] onResult: length=${result?.length}, value=${result?.take(200)}")
isExtracting = false isExtracting = false
statusMessage = null statusMessage = null
if (!result.isNullOrBlank() && result != "[]" && result != "null") { if (!result.isNullOrBlank() && result != "[]" && result != "null") {
@ -294,6 +305,12 @@ fun WebViewSyncSection(
statusMessage = "Keine Daten gefunden. Stelle sicher, dass du eingeloggt bist und sich der Verlauf geladen hat." statusMessage = "Keine Daten gefunden. Stelle sicher, dass du eingeloggt bist und sich der Verlauf geladen hat."
} }
} }
// Netflix: the script only clicks the download button; the result
// arrives via the DownloadListener, so stop the spinner shortly after
// unless a download has started.
if (provider == StreamingProvider.NETFLIX) {
statusMessage = "CSV-Download ausgelöst. Falls nichts passiert, lade die Seite neu."
}
}, },
colors = ButtonDefaults.buttonColors( colors = ButtonDefaults.buttonColors(
containerColor = provider.primaryColor, containerColor = provider.primaryColor,
@ -346,12 +363,6 @@ fun WebViewSyncSection(
super.onPageFinished(view, url) super.onPageFinished(view, url)
currentUrl = url ?: "" currentUrl = url ?: ""
isLoadingWeb = false isLoadingWeb = false
// Install the JS<->Kotlin bridge: the extraction promise resolves
// into window.__bingeResult, which runExtraction() then polls.
view?.evaluateJavascript(
"window.__bingeResult = null; window.__bingeExtracted = function(v) { window.__bingeResult = (typeof v === 'string') ? v : JSON.stringify(v); };",
null
)
} }
override fun shouldOverrideUrlLoading(view: WebView?, request: WebResourceRequest?): Boolean { override fun shouldOverrideUrlLoading(view: WebView?, request: WebResourceRequest?): Boolean {
@ -360,11 +371,25 @@ fun WebViewSyncSection(
} }
} }
loadUrl(provider.viewingHistoryUrl) loadUrl(provider.viewingHistoryUrl)
// Forward JS console.log to logcat so extraction can be debugged live.
webChromeClient = object : WebChromeClient() {
override fun onConsoleMessage(consoleMessage: android.webkit.ConsoleMessage): Boolean {
Log.d("BingeStats_Web", "[${consoleMessage.messageLevel()}] ${consoleMessage.message()}")
return true
}
}
loadedProvider = provider
webViewRef.value = this webViewRef.value = this
} }
}, },
// Removed the old aggressive reload: only react to provider changes, not every recomposition. // Reload only when the provider actually changes — not on every recomposition.
update = { _ -> }, update = { webView ->
if (loadedProvider != provider) {
isLoadingWeb = true
webView.loadUrl(provider.viewingHistoryUrl)
loadedProvider = provider
}
},
modifier = Modifier modifier = Modifier
.fillMaxSize() .fillMaxSize()
.border(1.dp, GlassCardBorder, RoundedCornerShape(8.dp)) .border(1.dp, GlassCardBorder, RoundedCornerShape(8.dp))
@ -381,58 +406,161 @@ fun WebViewSyncSection(
} }
/** /**
* Injects the provider-specific extraction script into the WebView. The Netflix script * Injects the provider-specific extraction script into the WebView. The scripts return a
* returns a Promise, so we wrap evaluation in a `.then()` and read the resolved JSON. * Promise; we attach a then()-handler that stores the resolved value, then poll the
* stored value until it appears (async fetch/scroll takes time).
*
* Bridge encoding: evaluateJavascript returns any string value JSON-encoded (i.e. wrapped
* in quotes and escaped). The stored __bingeResult is already a JSON string from the
* script, so when read back it arrives double-encoded. We unwrap exactly one layer.
*/ */
private fun runExtraction( private fun runExtraction(
webView: WebView, webView: WebView,
provider: StreamingProvider, provider: StreamingProvider,
onResult: (String?) -> Unit onResult: (String?) -> Unit
) { ) {
// Netflix: Netflix generates the CSV entirely client-side as a Blob, so there is no
// HTTP URL we can fetch with OkHttp. Instead we intercept the Blob right where it is
// created: we wrap document.createElement('a') so that when Netflix triggers the
// download, we read the Blob text via FileReader and store it. Then we click the
// "Download all" button and poll for the captured CSV text.
if (provider == StreamingProvider.NETFLIX) {
val setupAndClick = """
try {
function log(m){ try{ console.log('[BingeStats] '+m); }catch(e){} }
window.__bingeDone = false;
window.__bingeResult = null;
// Intercept the anchor-click that triggers the blob download. Netflix builds
// an <a download href="blob:..."> and clicks it. We wrap it to read the Blob.
if (!window.__bingeBlobPatched) {
window.__bingeBlobPatched = true;
var origCreate = document.createElement.bind(document);
document.createElement = function(tag) {
var el = origCreate(tag);
if (String(tag).toLowerCase() === 'a') {
var origClick = el.click.bind(el);
el.click = function() {
var href = el.href || el.getAttribute('href') || '';
if (href.indexOf('blob:') === 0 && el.href) {
try {
fetch(href).then(function(r){ return r.text(); }).then(function(txt){
log('Blob abgefangen: ' + txt.length + ' bytes');
window.__bingeResult = txt;
window.__bingeDone = true;
}).catch(function(e){ log('Blob-Lesefehler: ' + e.message); });
} catch (e) { log('Blob-Fetch Fehler: ' + e.message); }
}
return origClick();
};
}
return el;
};
}
// Find and click the download button.
var btns = document.querySelectorAll('button, a, [role="button"]');
var hit = null;
for (var i = 0; i < btns.length; i++) {
var t = (btns[i].textContent || '').trim().toLowerCase();
if (t.indexOf('alle herunterladen') !== -1 ||
t.indexOf('download all') !== -1 ||
t.indexOf('alle exportieren') !== -1 ||
t === 'export' || t === 'exportieren') {
hit = btns[i]; break;
}
}
if (hit) { log('Klicke Download-Button: ' + hit.textContent.trim()); hit.click(); }
else {
log('Kein Download-Button gefunden. Verfuegbare Buttons:');
document.querySelectorAll('button, a').forEach(function(b){
var tx=(b.textContent||'').trim(); if(tx.length>0&&tx.length<40) log(' -> "'+tx+'"');
});
}
} catch (e) { console.log('[BingeStats] Fehler: ' + e.message); }
""".trimIndent()
webView.evaluateJavascript(setupAndClick, null)
// The blob interception resolves asynchronously; poll for the captured CSV text.
pollExtraction(webView, attempts = 0, onResult = onResult)
return
}
val script = when (provider) { val script = when (provider) {
StreamingProvider.NETFLIX -> WebSyncScripts.netflix
StreamingProvider.AMAZON_PRIME -> WebSyncScripts.prime StreamingProvider.AMAZON_PRIME -> WebSyncScripts.prime
else -> { onResult(null); return } else -> { onResult(null); return }
} }
// The script string is itself an IIFE "(function(){...})()" that returns either a value
// or a Promise. We capture its return value directly and store the resolved/strungified
// result into window.__bingeResult, which pollExtraction then reads.
val wrapped = """ val wrapped = """
(function() { try {
var r = (function() { $script })(); var r = $script;
window.__bingeResult = null;
window.__bingeDone = false;
if (r && typeof r.then === 'function') { if (r && typeof r.then === 'function') {
r.then(function(v) { __bingeExtracted(v); }).catch(function() { __bingeExtracted('[]'); }); r.then(function(v) {
window.__bingeResult = (typeof v === 'string') ? v : JSON.stringify(v);
window.__bingeDone = true;
}).catch(function(e) {
window.__bingeResult = '[]';
window.__bingeDone = true;
});
} else { } else {
__bingeExtracted(r); window.__bingeResult = (typeof r === 'string') ? r : JSON.stringify(r);
window.__bingeDone = true;
}
} catch (e) {
window.__bingeResult = '[]';
window.__bingeDone = true;
} }
})();
""".trimIndent() """.trimIndent()
// Bridge from JS back to Kotlin: store the result in a JS variable we then retrieve.
webView.evaluateJavascript(wrapped, null) webView.evaluateJavascript(wrapped, null)
// Small delay so the async fetch (Netflix pagination) has time to complete before polling.
pollExtraction(webView, attempts = 0, onResult = onResult) pollExtraction(webView, attempts = 0, onResult = onResult)
} }
private fun pollExtraction(webView: WebView, attempts: Int, onResult: (String?) -> Unit) { private fun pollExtraction(webView: WebView, attempts: Int, onResult: (String?) -> Unit) {
// The injected promise calls __bingeExtracted(v); we expose it via addJavascriptInterface-like // Clicking through thousands of "Show more" pages takes a while, so allow generous
// retrieval: evaluate a getter. Cap attempts so we never spin forever. // polling time before giving up.
val maxAttempts = 20 val maxAttempts = 300
if (attempts > maxAttempts) { if (attempts > maxAttempts) {
onResult(null) onResult(null)
return return
} }
webView.postDelayed({ webView.postDelayed({
webView.evaluateJavascript( webView.evaluateJavascript(
"(window.__bingeResult === undefined) ? null : window.__bingeResult;" "window.__bingeDone ? window.__bingeResult : null;"
) { raw -> ) { raw ->
if (raw == null || raw == "null" || raw == "undefined") { android.util.Log.d("BingeStats_Web", "[Kotlin] poll attempt=$attempts raw=${raw?.take(120)}")
// evaluateJavascript returns the value JSON-encoded: a string arrives as
// "\"...\"" and null as "null". Unwrap one JSON string layer.
val unwrapped = unwrapJsString(raw)
if (unwrapped == null) {
pollExtraction(webView, attempts + 1, onResult) pollExtraction(webView, attempts + 1, onResult)
} else { } else {
onResult(raw) onResult(unwrapped)
} }
} }
}, 500) }, 500)
} }
/**
* evaluateJavascript returns JSON-encoded values. A JS string "abc" comes back as
* "\"abc\"". null/undefined come back as "null". This unwraps the outer JSON quotes for
* a string and returns null for actual null/undefined.
*/
private fun unwrapJsString(raw: String?): String? {
if (raw == null || raw == "null" || raw == "undefined") return null
val trimmed = raw.trim()
// Already a JSON string literal -> unwrap one quoting layer.
if (trimmed.startsWith("\"") && trimmed.endsWith("\"")) {
return trimmed.substring(1, trimmed.length - 1)
.replace("\\\"", "\"")
.replace("\\\\", "\\")
.replace("\\n", "\n")
}
return trimmed
}
@Composable @Composable
fun CsvImportSection( fun CsvImportSection(
provider: StreamingProvider, provider: StreamingProvider,