I am a native Android developer considering migrating to Flutter, I have done all the research for the core library alternatives in Flutter. One thing I am particular about is that when When working with a list of large data. In Android, we can persist this data in an SQLite local DB using the ROOM library. What's even better is that with the RemoteMediator class available in Paging 3 Android library, we can create an infinite scrolling recycler while querying data from the local DB while the network call queries new data and stores it in the DB. So the recycler queries data from the database not from a network call. So this data can be access without internet access.
I know that the sqflite package is Flutter's alternative to ROOM in Android but can we use this database to query a paged list of items to display in the ListViewBuilder and the user scrolls?
Ive just been doing this and couldn't figure it out at first. But have now a working solution. I use a database first approach so my database is my source of truth and the information that populates it comes from our api the portal.
I use the infinite scrolling package for my pagination and every time it fetches a new page I sync my database hive with the results, I then return a sublist of the database.
The sync is an upsert so it updates records that already exist and adds records that don't.
The infinite scroll package allows us to implement a few things, the first is a page request listener this tells us what page we are on, we need to increment this in our fetchPage method.
Heres a great article on it here
Second is a paging controller, this is what allows us to either append a page or append a last page, so we will need to figure out when the last page is.
The other small gotcha is managing the small discrepancy between what's in your database and what's in the portal. For example if my database starts with 17 entries that don't exist yet in the portal, and then I start requesting pages, when I request the last page and it only gives me for example 5 results I need to know to add this on to the remaining 17 extra entries in the database this makes a total of 22 which means my last page would be different, we need to calculate this to not show duplicate records and not miss any items.
Enough talking show me the code.
Here is my adapter
import 'dart:math' as math;
import 'package:built_collection/built_collection.dart';
import 'package:infinite_scroll_pagination/infinite_scroll_pagination.dart';
import '../api/models/error_response.dart';
import '../api/models/events/event.dart';
import '../api/models/events/event_paging_response.dart';
import '../api/models/util/pagination.dart';
import '../api/models/util/portal_request.dart';
import '../api/utils/api_response.dart';
import '../api/utils/app_exceptions.dart';
import '../extensions/iterable_extension.dart';
import '../repositories/hive_event_repository.dart';
import '../repositories/event_repository.dart';
import '../repositories/event_sync_repository.dart';
class EventPagingAdapter {
EventPagingAdapter(
this.eventSyncRepository,
this.databaseEventRepository,
this.eventRepository,
);
final EventSyncRepository eventSyncRepository;
final EventRepository eventRepository;
final DatabaseEventRepository databaseEventRepository;
PagingController<int, Event>? pagingController;
Pagination? _lastPageInPortal;
Pagination? _lastPortalResponse;
bool _hasFetchedOnce = false;
bool _hasBeenDisposed = false;
int get limit => 20;
PortalRequest? _eventRequest;
PortalRequest? get eventRequest => _eventRequest;
void initialize() {
pagingController?.addPageRequestListener((pageKey) {
_requestPage(_lastPortalResponse, pageKey);
});
}
void setEventRequest(
PortalRequest eventRequest,
) {
if (_eventRequest == null) {
_eventRequest = eventRequest;
} else {
_eventRequest = _eventRequest?.rebuild(
(b) => b..search = eventRequest.search,
);
}
}
void refresh() {
if (!_hasFetchedOnce) {
_hasFetchedOnce = true;
_firstRequest();
} else {
pagingController?.refresh();
}
}
void retryLastRequest() {
pagingController?.retryLastFailedRequest();
}
void dispose() {
_hasBeenDisposed = true;
pagingController?.dispose();
}
Future<void> _requestPage(
Pagination? _lastPortalResponse,
int pageKey,
) async {
final _notNullEventRequest = _eventRequest;
if (_notNullEventRequest != null) {
EventPagingResponse? _portalPage;
try {
if (_lastPageInPortal == null && !_isLastPortalPage(_lastPortalResponse)) {
_portalPage = await _fetchPortalPage(
_notNullEventRequest,
pageKey,
);
await _syncEvents(
_portalPage?.data ?? _buildEmptyBuiltList(),
);
} else if (_isLastPortalPage(_lastPortalResponse)) {
_lastPageInPortal = _lastPortalResponse;
}
await _fetchAndAppendDBPage(
_portalPage,
pageKey,
);
} catch (error) {
if (!_hasBeenDisposed) {
_handleError(
error,
_portalPage,
pageKey,
);
}
}
}
}
Future<void> _fetchAndAppendDBPage(
EventPagingResponse? _portalPage,
int pageKey,
) async {
if (!_hasBeenDisposed) {
final page = await _fetchDBPage(
pageKey,
_portalPage?.pagination ?? _lastPageInPortal,
);
_appendPage(
page,
_portalPage?.pagination ?? _lastPageInPortal,
pageKey,
);
}
}
Future<EventPagingResponse?> _fetchPortalPage(
PortalRequest _eventRequest,
int pageKey,
) async {
final _skip = pageKey * limit;
return eventRepository.getEventsResponse(
eventRequest: _eventRequest.rebuild(
(p0) => p0
..limit = limit
..skip = _skip
..sort = 'booking_start_date'
..pagination = true,
),
);
}
Future<EventPagingResponse?> _fetchDBPage(
int pageKey,
Pagination? _portalPage,
) async {
final _lastPageInPortalCount = _portalPage?.count ?? 0;
final _skip = _portalPage != null ? pageKey * limit - (limit - _lastPageInPortalCount) : pageKey * limit;
return _getPagedDatabaseEventList(
_portalPage != null
? _eventRequest?.rebuild(
(p0) => p0..skip = _portalPage.skip,
)
: _eventRequest?.rebuild(
(p0) => p0..skip = _skip,
),
pageKey,
);
}
Future<void> _appendPage(
EventPagingResponse? hivePage,
Pagination? _portalPagination,
int pageKey,
) async {
final _isLastDBPageBackingField = await _isLastDBPage(
hivePage?.pagination,
);
//_portalPagination will be null if the last request failed, for instance we may be offline
if (_portalPagination == null && _isLastDBPageBackingField ||
_isLastPortalPage(_portalPagination) && _isLastDBPageBackingField) {
pagingController?.appendLastPage(
hivePage?.data.toList() ?? [],
);
} else {
final nextPageKey = pageKey + 1;
pagingController?.appendPage(
hivePage?.data.toList() ?? [],
nextPageKey,
);
}
}
Future<void> _syncEvents(
BuiltList<Event> eventList,
) async {
if (eventList.isNotEmpty) {
await databaseEventRepository.syncEvents(
eventList,
);
}
await eventSyncRepository.removeEvents();
}
Future<EventPagingResponse> _getPagedDatabaseEventList(
PortalRequest? _eventRequest,
int pageKey,
) async {
final _skip = _eventRequest?.skip ?? 0;
final _eventList = await _getDatabaseEventList(
_eventRequest?.search,
);
final _hiveTotal = _eventList.length;
final _count = math.min(_hiveTotal - _skip, limit).clamp(0, _hiveTotal);
final _end = _count + _skip;
return _buildEventPagingResponse(
_eventList.sublist(_skip, _end).toBuiltList(),
_buildPagination(
_eventRequest,
_count,
_hiveTotal,
),
);
}
Pagination _buildPagination(
PortalRequest? _eventRequest,
int _count,
int _hiveTotal,
) {
return Pagination(
(b) => b
..skip = _eventRequest?.skip
..count = _count
..limit = limit
..total = _hiveTotal,
);
}
EventPagingResponse _buildEventPagingResponse(
BuiltList<Event> data,
Pagination pagination,
) {
return EventPagingResponse(
(b) => b
..fats = null
..pagination = pagination.toBuilder()
..data = data.toBuilder(),
);
}
void _firstRequest() {
_requestPage(null, 0);
}
Future<int> _getEventCount(
String? search,
) async {
final eventCount = await databaseEventRepository.getEventCount(
search: search?.replaceAll(' ', '') ?? '',
);
return eventCount;
}
Future<List<Event>> _getDatabaseEventList(
String? search,
) async {
final eventList = await databaseEventRepository.getEvents(
search: search?.replaceAll(' ', '') ?? '',
);
return eventList.uniqueBy((e) => e.booking_ref).toList();
}
bool _isLastPortalPage(Pagination? newPage) {
if (newPage == null) {
return false;
}
final _skip = newPage.skip ?? 0;
final _count = newPage.count ?? 0;
final _total = newPage.total ?? 0;
return _skip + _count >= _total;
}
Future<bool> _isLastDBPage(
Pagination? newPage,
) async {
final _skip = newPage?.skip ?? 0;
final _count = newPage?.count ?? 0;
final _hiveTotal = await _getEventCount(
_eventRequest?.search,
);
return _skip + _count >= _hiveTotal;
}
Future<void> _handleError(
Object error,
EventPagingResponse? _portalPage,
int pageKey,
) async {
try {
final appException = error as AppException;
if (appException.error?.statusCode == 503) {
await _fetchAndAppendDBPage(
_portalPage,
pageKey,
);
} else {
pagingController?.error = ApiResponse.error(
appException.error?.message,
error: ErrorResponse(
(b) => b
..message = appException.error?.message
..error = error.toString()
..url = appException.error?.url,
),
);
}
} catch (e) {
pagingController?.error = ApiResponse.error(
'Application Error',
error: ErrorResponse(
(b) => b
..statusCode = -1
..message = '$error',
),
);
}
}
BuiltList<Event> _buildEmptyBuiltList() {
return BuiltList.of(
[],
);
}
}
We control all this from a view model
import 'package:infinite_scroll_pagination/infinite_scroll_pagination.dart';
import 'package:rxdart/rxdart.dart';
import '../../adapters/timesheet_paging_adapter.dart';
import '../../api/models/events/event.dart';
import '../../api/models/util/portal_request.dart';
class EventListViewModel {
EventListViewModel(this._eventPagingAdapter);
final EventPagingAdapter _eventPagingAdapter;
final _pagingController = PagingController<int, Timesheet>(firstPageKey: 0);
var _isPagingAdapterInitialized = false;
final searchText = BehaviorSubject<String?>.seeded('');
void _initializePagingAdapter() {
_eventPagingAdapter.pagingController = _pagingController;
_eventPagingAdapter.initialize();
}
void refresh() {
_eventPagingAdapter.refresh();
}
void updateQuery(PortalRequest portalRequest) {
_eventPagingAdapter.setTimesheetRequest(portalRequest);
refresh();
}
void retryLastRequest() {
_eventPagingAdapter.pagingController?.retryLastFailedRequest();
}
PagingController<int, Timesheet> getPagingController() {
if (_isPagingAdapterInitialized == false) {
_isPagingAdapterInitialized = true;
_initializePagingAdapter();
}
return _eventPagingAdapter.pagingController ?? _pagingController;
}
void dispose() {
searchText.close();
_eventPagingAdapter.dispose();
}
}
And that populates our Event views PagedSliverList from the infinite scroll package.
import 'package:flutter/material.dart';
import 'package:infinite_scroll_pagination/infinite_scroll_pagination.dart';
import 'package:sliver_tools/sliver_tools.dart';
import '../../api/models/events/event.dart';
import '../../api/utils/api_response.dart';
import '../../view_models/events/event_list_view_model.dart';
import '../shared_widgets/connectivity_widget.dart';
import '../shared_widgets/error_widget.dart' as ew;
import '../shared_widgets/no_results.dart';
import '../shared_widgets/rounded_button.dart';
import '../shared_widgets/my_loading_widget.dart';
import '../shared_widgets/my_sliver_refresh_indicator.dart';
import 'event_tile.dart';
class EventListView extends StatelessWidget {
const EventListView({
Key? key,
required this.eventListViewModel,
}) : super(key: key);
final EventListViewModel eventListViewModel;
@override
Widget build(BuildContext context) {
return MySliverRefreshIndicator(
onRefresh: eventListViewModel.refresh,
padding: EdgeInsets.zero,
sliver: MultiSliver(
children: [
SliverPadding(
padding: const EdgeInsets.all(8),
sliver: PagedSliverList.separated(
pagingController: eventListViewModel.getPagingController(),
builderDelegate: PagedChildBuilderDelegate<Event>(
itemBuilder: (context, timesheet, index) => _eventItem(
timesheet: timesheet,
),
firstPageErrorIndicatorBuilder: (context) => _buildErrorWidget(),
noItemsFoundIndicatorBuilder: (context) => _emptyListIndicator(),
newPageErrorIndicatorBuilder: (context) =>
_errorListItemWidget(onTryAgain: eventListViewModel.retryLastRequest),
firstPageProgressIndicatorBuilder: (context) => const Center(
child: MyLoadingWidget(),
),
newPageProgressIndicatorBuilder: (context) => _loadingListItemWidget(),
),
separatorBuilder: (context, index) => const SizedBox(
height: 4,
),
),
),
],
),
);
}
ew.ErrorWidget _buildErrorWidget() {
final error = eventListViewModel.getPagingController().error as ApiResponse;
return ew.ErrorWidget(
showImage: true,
error: error,
onTryAgain: () => eventListViewModel.getPagingController().refresh(),
);
}
Widget _errorListItemWidget({
required VoidCallback onTryAgain,
}) {
return Column(
mainAxisSize: MainAxisSize.min,
children: [
const Text('Error getting new page...'),
RoundedButton(
label: 'Retry',
onPressed: onTryAgain,
),
],
);
}
Widget _loadingListItemWidget() {
return const SizedBox(
height: 36,
child: Center(
child: MyLoadingWidget(),
),
);
}
Widget _emptyListIndicator() {
return const NoResults();
}
Widget _eventItem({required Event event}) {
return EventTile(
event: event,
refreshBookings: eventListViewModel.getPagingController().refresh,
);
}
}
Much of this code has been obfuscated.
If anyone see's any obvious flaws in the logic or other issues please let me know.