How to make pagination work for firestore in realtime on Android?

Question

I went through Doug Stevenson's solution to pagination for Firestore using Android architecture patterns present at https://github.com/CodingDoug/firebase-jetpack. My concern is that how do I make this work in realtime using Android architecture components. I attempted a solution which is described as follows but there are some issues with it. I tried to make pagination work in realtime by creating a FirestoreBoundaryCallback:-

class FirestoreBoundaryCallback<T>(
    private val baseQuery: Query,
    private val factory: FirestoreQueryDataSource.Factory,
    private val lifecycleOwner: LifecycleOwner): PagedList.BoundaryCallback<QueryItemOrException<T>>() {

private val allLiveData = mutableListOf<FirebaseQueryLiveData>()
private val mutableLoadingState = MutableLiveData<LoadingState>()

val loadingState: LiveData<LoadingState>
    get() = mutableLoadingState

override fun onZeroItemsLoaded() {
    allLiveData.clear()
    mutableLoadingState.value = LoadingState.LOADING_INITIAL
    val query = baseQuery.limit(50)
    val liveData = FirebaseQueryLiveData(query)
    liveData.observe(lifecycleOwner, Observer {
        mergeAllDocs()
        if (mutableLoadingState.value != LoadingState.LOADED) {
            mutableLoadingState.value = LoadingState.LOADED
        }
    })
    allLiveData.add(liveData)
}

override fun onItemAtEndLoaded(itemAtEnd: QueryItemOrException<T>) {
    if (allLiveData.isNotEmpty() && allLiveData.last().value?.data?.documents?.isNotEmpty() == true) {
        val lastDocument = allLiveData.last().value?.data?.documents?.last()
        if (lastDocument != null) {
            val query = baseQuery.startAfter(lastDocument).limit(50)
            val liveData = FirebaseQueryLiveData(query)
            mutableLoadingState.value = LoadingState.LOADING_MORE
            liveData.observe(lifecycleOwner, Observer {
                mergeAllDocs()
                if (mutableLoadingState.value != LoadingState.LOADED) {
                    mutableLoadingState.value = LoadingState.LOADED
                }
            })
            allLiveData.add(liveData)
        }
    }
}

fun mergeAllDocs() {
    val items = mutableListOf<DocumentSnapshot>()
    allLiveData.forEach{
        val docs = it.value?.data?.documents
        if (docs != null) {
            items.addAll(docs)
        }
    }
    factory.setItems(items)
}

override fun onItemAtFrontLoaded(itemAtFront: QueryItemOrException<T>) {
}
}

And then I modified the FirestoreQueryDataSource in the following way:-

class FirestoreQueryDataSource private constructor(
private val documentSnapshots: List<DocumentSnapshot>
) : PageKeyedDataSource<PageKey, DocumentSnapshot>() {

companion object {
    private const val TAG = "FirestoreQueryDataSrc"
}

class Factory(private val query: Query, private val source: Source) : DataSource.Factory<PageKey, DocumentSnapshot>() {
    val sourceLiveData = MutableLiveData<FirestoreQueryDataSource>()
    var documentSnapshots: List<DocumentSnapshot> = mutableListOf()

    fun setItems(items: List<DocumentSnapshot>) {
        sourceLiveData.value?.invalidate()
        documentSnapshots = items
        sourceLiveData.postValue(FirestoreQueryDataSource(documentSnapshots))
    }

    override fun create(): DataSource<PageKey, DocumentSnapshot> {
        val dataSource = FirestoreQueryDataSource(documentSnapshots)
        sourceLiveData.postValue(dataSource)
        return dataSource
    }
}

override fun loadInitial(
        params: LoadInitialParams<PageKey>,
        callback: LoadInitialCallback<PageKey, DocumentSnapshot>) {

    val firstPageDocSnapshots = documentSnapshots.take(params.requestedLoadSize)
    val nextPageKey = getNextPageKey(firstPageDocSnapshots)
    callback.onResult(firstPageDocSnapshots, null, nextPageKey)
}

override fun loadAfter(
        params: LoadParams<PageKey>,
        callback: LoadCallback<PageKey, DocumentSnapshot>) {

    val startAfterIndex = documentSnapshots.indexOf(params.key.startAfterDoc)
    var endIndex = startAfterIndex + params.requestedLoadSize
    if (endIndex > documentSnapshots.size) {
        endIndex = documentSnapshots.size - 1;
    }
    val afterInitialPageDocs = documentSnapshots.subList(startAfterIndex, endIndex)
    val nextPageKey = getNextPageKey(afterInitialPageDocs)
    callback.onResult(afterInitialPageDocs, nextPageKey)
}

override fun loadBefore(
        params: LoadParams<PageKey>,
        callback: LoadCallback<PageKey, DocumentSnapshot>) {
    // The paging here only understands how to append new items to the
    // results, not prepend items from earlier pages.
    callback.onResult(emptyList(), null)
}

private fun getNextPageKey(documents: List<DocumentSnapshot>): PageKey? {
    return if (documents.isNotEmpty()) {
        PageKey(documents.last())
    } else {
        null
    }
}
}

data class PageKey(val startAfterDoc: DocumentSnapshot)

In my ViewModel, this is what I return:-

val sourceFactory = FirestoreQueryDataSource.Factory(query, Source.DEFAULT)
        val deserializedDataSourceFactory = sourceFactory.map { snapshot ->
            try {
                val item = QueryItem(Deserializer.deserialize(snapshot, Record::class.java), snapshot.id)
                item.item.id = snapshot.id
                QueryItemOrException(item, null)
            } catch (e: Exception) {
                Log.e(TAG, "Error while deserializing order", e)
                QueryItemOrException<PosOrder>(null, e)
            }
        }
        val boundaryCallback = FirestoreBoundaryCallback<Record>(query, sourceFactory, lifecycleOwner)
        val livePagedList = LivePagedListBuilder(deserializedDataSourceFactory, 30)
                .setFetchExecutor(executors.cpuExecutorService)
                .setBoundaryCallback(boundaryCallback)
                .build()
        return Listing(
                pagedList = livePagedList,
                loadingState = boundaryCallback.loadingState,
                refresh = {
                    sourceFactory.sourceLiveData.value?.invalidate()
                }
        )

The performance is not good and also I am concerned that when a new record is inserted, the first page will lose its last record (since the limit is fixed for the first query) and second page will continue to start AFTER the last record of the older first page. What would be the correct way of paginating firestore data on Android with realtime updates?

Answer 1

The limitation of the the Paging architecture component is that you can't also get realtime updates at the same time. You have to choose if you want realtime updates or paging. (Or come up with your own solution entirely that doesn't using Paging or LiveData). This is because of the way the Paging component works. It only deals with sets of static data retrieved by paging the one-time queries to the actual data.