diff --git a/lib/friendica_client/friendica_client.dart b/lib/friendica_client/friendica_client.dart index 867bdcc..f1861a3 100644 --- a/lib/friendica_client/friendica_client.dart +++ b/lib/friendica_client/friendica_client.dart @@ -6,6 +6,7 @@ import 'package:http_parser/http_parser.dart'; import 'package:logging/logging.dart'; import 'package:result_monad/result_monad.dart'; +import '../friendica_client/paged_response.dart'; import '../globals.dart'; import '../models/TimelineIdentifiers.dart'; import '../models/connection.dart'; @@ -25,6 +26,7 @@ import '../serializers/mastodon/group_data_mastodon_extensions.dart'; import '../serializers/mastodon/notification_mastodon_extension.dart'; import '../serializers/mastodon/timeline_entry_mastodon_extensions.dart'; import '../services/auth_service.dart'; +import 'paging_data.dart'; class FriendicaClient { static final _logger = Logger('$FriendicaClient'); @@ -43,13 +45,14 @@ class FriendicaClient { _authHeader = "Basic $encodedAuthString"; } + // TODO Convert Notifications to using paging for real FutureResult, ExecError> getNotifications() async { final url = 'https://$serverName/api/v1/notifications?include_all=true&limit=200'; final request = Uri.parse(url); _logger.finest(() => 'Getting new notifications'); return (await _getApiListRequest(request).andThenSuccessAsync( - (notificationsJson) async => notificationsJson + (notificationsJson) async => notificationsJson.data .map((json) => NotificationMastodonExtension.fromJson(json)) .toList())) .mapError((error) { @@ -79,12 +82,13 @@ class FriendicaClient { return response.mapValue((value) => true); } + // TODO Convert Albums to using paging for real FutureResult, ExecError> getGalleryData() async { _logger.finest(() => 'Getting gallery data'); final url = 'https://$serverName/api/friendica/photoalbums'; final request = Uri.parse(url); return (await _getApiListRequest(request).andThenSuccessAsync( - (albumsJson) async => albumsJson + (albumsJson) async => albumsJson.data .map((json) => GalleryDataFriendicaExtensions.fromJson(json)) .toList())) .mapError((error) => error is ExecError @@ -92,6 +96,7 @@ class FriendicaClient { : ExecError(type: ErrorType.localError, message: error.toString())); } + // TODO Convert Gallery Images to using paging for real FutureResult, ExecError> getGalleryImages( String galleryName) async { _logger.finest(() => 'Getting gallery data'); @@ -99,7 +104,7 @@ class FriendicaClient { 'https://$serverName/api/friendica/photoalbum?album=$galleryName'; final request = Uri.parse(url); return (await _getApiListRequest(request).andThenSuccessAsync( - (imagesJson) async => imagesJson + (imagesJson) async => imagesJson.data .map((json) => ImageEntryFriendicaExtension.fromJson(json)) .toList())) .mapError((error) => error is ExecError @@ -107,12 +112,13 @@ class FriendicaClient { : ExecError(type: ErrorType.localError, message: error.toString())); } + // TODO Convert Groups to using paging for real (if it is supported) FutureResult, ExecError> getGroups() async { _logger.finest(() => 'Getting group (Mastodon List) data'); final url = 'https://$serverName/api/v1/lists'; final request = Uri.parse(url); return (await _getApiListRequest(request).andThenSuccessAsync( - (listsJson) async => listsJson + (listsJson) async => listsJson.data .map((json) => GroupDataMastodonExtensions.fromJson(json)) .toList())) .mapError((error) => error is ExecError @@ -146,33 +152,34 @@ class FriendicaClient { return (await _deleteUrl(request, requestData)).mapValue((_) => true); } - FutureResult, ExecError> getMyFollowing( - {int sinceId = -1, int maxId = -1, int limit = 50}) async { - _logger.finest(() => - 'Getting following data since $sinceId, maxId $maxId, limit $limit'); + FutureResult>, ExecError> getMyFollowing( + PagingData page) async { + _logger.finest(() => 'Getting following with paging data $page'); final myId = getIt().currentId; - final paging = - _buildPagingData(sinceId: sinceId, maxId: maxId, limit: limit); final baseUrl = 'https://$serverName/api/v1/accounts/$myId'; - return (await _getApiListRequest(Uri.parse('$baseUrl/following&$paging')) - .andThenSuccessAsync((listJson) async => listJson - .map((json) => ConnectionMastodonExtensions.fromJson(json)) - .toList())) + final result = await _getApiListRequest( + Uri.parse('$baseUrl/following?${page.toQueryParameters()}'), + ); + return result + .andThenSuccess((response) => response.map((jsonArray) => jsonArray + .map((json) => ConnectionMastodonExtensions.fromJson(json)) + .toList())) .execErrorCast(); } - FutureResult, ExecError> getMyFollowers( - {int sinceId = -1, int maxId = -1, int limit = 50}) async { - _logger.finest(() => - 'Getting followers data since $sinceId, maxId $maxId, limit $limit'); + FutureResult>, ExecError> getMyFollowers( + PagingData page) async { + _logger.finest(() => 'Getting followers data with page data $page'); final myId = getIt().currentId; - final paging = - _buildPagingData(sinceId: sinceId, maxId: maxId, limit: limit); final baseUrl = 'https://$serverName/api/v1/accounts/$myId'; - return (await _getApiListRequest(Uri.parse('$baseUrl/followers&$paging')) - .andThenSuccessAsync((listJson) async => listJson - .map((json) => ConnectionMastodonExtensions.fromJson(json)) - .toList())) + final result1 = await _getApiListRequest( + Uri.parse('$baseUrl/followers&${page.toQueryParameters()}'), + ); + + return result1 + .andThenSuccess((response) => response.map((jsonArray) => jsonArray + .map((json) => ConnectionMastodonExtensions.fromJson(json)) + .toList())) .execErrorCast(); } @@ -185,14 +192,14 @@ class FriendicaClient { final baseUrl = 'https://$serverName/api/v1/accounts/$myId'; final following = await _getApiListRequest(Uri.parse('$baseUrl/following$paging')).fold( - onSuccess: (followings) => followings.isNotEmpty, + onSuccess: (followings) => followings.data.isNotEmpty, onError: (error) { _logger.severe('Error getting following list: $error'); return false; }); final follower = await _getApiListRequest(Uri.parse('$baseUrl/followers$paging')).fold( - onSuccess: (followings) => followings.isNotEmpty, + onSuccess: (followings) => followings.data.isNotEmpty, onError: (error) { _logger.severe('Error getting follower list: $error'); return false; @@ -209,6 +216,7 @@ class FriendicaClient { return Result.ok(connection.copy(status: status)); } + // TODO Convert groups for connection to using paging for real (if available) FutureResult, ExecError> getMemberGroupsForConnection( String connectionId) async { _logger.finest(() => @@ -216,12 +224,13 @@ class FriendicaClient { final url = 'https://$serverName/api/v1/accounts/$connectionId/lists'; final request = Uri.parse(url); return (await _getApiListRequest(request).andThenSuccessAsync( - (listsJson) async => listsJson + (listsJson) async => listsJson.data .map((json) => GroupDataMastodonExtensions.fromJson(json)) .toList())) .mapError((error) => error as ExecError); } + // TODO Convert User Timeline to using paging for real FutureResult, ExecError> getUserTimeline( {String userId = '', int page = 1, int count = 10}) async { _logger.finest(() => 'Getting user timeline for $userId'); @@ -232,29 +241,23 @@ class FriendicaClient { : '${baseUrl}screen_name=$userId$pagingData'; final request = Uri.parse(url); return (await _getApiListRequest(request).andThenSuccessAsync( - (postsJson) async => postsJson + (postsJson) async => postsJson.data .map((json) => TimelineEntryMastodonExtensions.fromJson(json)) .toList())) .mapError((error) => error as ExecError); } FutureResult, ExecError> getTimeline( - {required TimelineIdentifiers type, - int sinceId = 0, - int maxId = 0, - int limit = 20}) async { + {required TimelineIdentifiers type, required PagingData page}) async { final String timelinePath = _typeToTimelinePath(type); final String timelineQPs = _typeToTimelineQueryParameters(type); final baseUrl = 'https://$serverName/api/v1/$timelinePath'; - final pagingData = - _buildPagingData(sinceId: sinceId, maxId: maxId, limit: limit); - - final url = '$baseUrl?exclude_replies=true&$pagingData&$timelineQPs'; + final url = + '$baseUrl?exclude_replies=true&${page.toQueryParameters()}&$timelineQPs'; final request = Uri.parse(url); - _logger.finest(() => - 'Getting ${type.toHumanKey()} limit $limit sinceId: $sinceId maxId: $maxId : $url'); + _logger.finest(() => 'Getting ${type.toHumanKey()} with paging data $page'); return (await _getApiListRequest(request).andThenSuccessAsync( - (postsJson) async => postsJson + (postsJson) async => postsJson.data .map((json) => TimelineEntryMastodonExtensions.fromJson(json)) .toList())) .execErrorCast(); @@ -282,6 +285,7 @@ class FriendicaClient { } } + // TODO Convert getPostOrComment to using paging for real FutureResult, ExecError> getPostOrComment(String id, {bool fullContext = false}) async { return (await runCatchingAsync(() async { @@ -306,10 +310,7 @@ class FriendicaClient { } })); })) - .mapError((error) => ExecError( - type: ErrorType.parsingError, - message: error.toString(), - )); + .execErrorCastAsync(); } FutureResult deleteEntryById(String id) async { @@ -515,7 +516,7 @@ class FriendicaClient { return Result.ok(newImageData); } - FutureResult _getUrl(Uri url) async { + FutureResult, ExecError> _getUrl(Uri url) async { _logger.finer('GET: $url'); try { final response = await http.get( @@ -531,7 +532,10 @@ class FriendicaClient { type: ErrorType.authentication, message: '${response.statusCode}: ${response.reasonPhrase}')); } - return Result.ok(utf8.decode(response.bodyBytes)); + return PagedResponse.fromLinkHeader( + response.headers['link'], + utf8.decode(response.bodyBytes), + ); } catch (e) { return Result.error( ExecError(type: ErrorType.localError, message: e.toString())); @@ -589,16 +593,20 @@ class FriendicaClient { } } - FutureResult, ExecError> _getApiListRequest(Uri url) async { + FutureResult>, ExecError> _getApiListRequest( + Uri url) async { return (await _getUrl(url).andThenSuccessAsync( - (jsonText) async => jsonDecode(jsonText) as List)) + (response) async => + response.map((data) => jsonDecode(data) as List), + )) .mapError((error) => error as ExecError); } FutureResult _getApiRequest(Uri url) async { - return (await _getUrl(url) - .andThenSuccessAsync((jsonText) async => jsonDecode(jsonText))) - .mapError((error) => error as ExecError); + return (await _getUrl(url).andThenSuccessAsync( + (response) async => jsonDecode(response.data), + )) + .execErrorCastAsync(); } String _typeToTimelinePath(TimelineIdentifiers type) { @@ -632,20 +640,6 @@ class FriendicaClient { } } - String _buildPagingData( - {required int sinceId, required int maxId, required int limit}) { - var pagingData = 'limit=$limit'; - if (maxId > 0) { - pagingData = '$pagingData&max_id=$maxId'; - } - - if (sinceId > 0) { - pagingData = '&since_id=$sinceId'; - } - - return pagingData; - } - Connection _updateConnectionFromFollowRequestResult( Connection connection, String jsonString) { final json = jsonDecode(jsonString) as Map; diff --git a/lib/friendica_client/paged_response.dart b/lib/friendica_client/paged_response.dart new file mode 100644 index 0000000..af6aef4 --- /dev/null +++ b/lib/friendica_client/paged_response.dart @@ -0,0 +1,97 @@ +import 'package:logging/logging.dart'; +import 'package:result_monad/result_monad.dart'; + +import '../models/exec_error.dart'; +import 'paging_data.dart'; + +final _logger = Logger('PagedResponse'); + +class PagedResponse { + PagingData? previous; + PagingData? next; + T data; + + PagedResponse(this.data, {this.previous, this.next}); + + bool get hasMorePages => previous != null || next != null; + + static Result, ExecError> fromLinkHeader( + String? linkHeader, T data) { + if (linkHeader == null || linkHeader.isEmpty) { + return Result.ok(PagedResponse(data)); + } + + String? previousPage; + String? nextPage; + for (String linkTerms in linkHeader.trim().split(',')) { + if (linkHeader.isEmpty) { + return buildErrorResult( + type: ErrorType.parsingError, + message: 'Link header element is blank', + ); + } + final paging = linkTerms.split(';'); + if (paging.length != 2) { + return buildErrorResult( + type: ErrorType.parsingError, + message: + 'Incorrect number of elements, ${paging.length} != 2, for: $linkTerms', + ); + } + final urlPieceString = paging.first.trim(); + if (!urlPieceString.startsWith('<') && !urlPieceString.endsWith('>')) { + return buildErrorResult( + type: ErrorType.parsingError, + message: + 'Link URL is malformed (no leading trailing <>): $urlPieceString', + ); + } + final url = urlPieceString.substring(1, urlPieceString.length - 1); + final directionString = paging.last.trim(); + if (directionString == 'rel="prev"') { + previousPage = url; + } else if (directionString == 'rel="next"') { + nextPage = url; + } else { + _logger.info('Unknown paging data: $directionString for url: $url'); + } + } + + return Result.ok(PagedResponse( + data, + previous: previousPage == null + ? null + : PagingData.fromQueryParameters( + Uri.parse(previousPage), + ), + next: nextPage == null + ? null + : PagingData.fromQueryParameters( + Uri.parse(nextPage), + ), + )); + } + + PagedResponse map(T2 Function(T data) func) => PagedResponse( + func(data), + previous: previous, + next: next, + ); + + @override + String toString() { + return 'PagedResponse{previous: $previous, next: $next, data: $data}'; + } + + @override + bool operator ==(Object other) => + identical(this, other) || + other is PagedResponse && + runtimeType == other.runtimeType && + previous == other.previous && + next == other.next && + data == other.data; + + @override + int get hashCode => previous.hashCode ^ next.hashCode ^ data.hashCode; +} diff --git a/lib/friendica_client/paging_data.dart b/lib/friendica_client/paging_data.dart new file mode 100644 index 0000000..f0565cf --- /dev/null +++ b/lib/friendica_client/paging_data.dart @@ -0,0 +1,64 @@ +class PagingData { + static const DEFAULT_LIMIT = 50; + + final int? minId; + final int? maxId; + final int? sinceId; + final int limit; + + PagingData({ + this.minId, + this.maxId, + this.sinceId, + this.limit = DEFAULT_LIMIT, + }); + + factory PagingData.fromQueryParameters(Uri uri) { + final minIdString = uri.queryParameters['min_id']; + final maxIdString = uri.queryParameters['max_id']; + final sinceIdString = uri.queryParameters['since_id']; + final limitString = uri.queryParameters['limit']; + return PagingData( + minId: int.tryParse(minIdString ?? ''), + maxId: int.tryParse(maxIdString ?? ''), + sinceId: int.tryParse(sinceIdString ?? ''), + limit: int.tryParse(limitString ?? '') ?? DEFAULT_LIMIT, + ); + } + + String toQueryParameters() { + var pagingData = 'limit=$limit'; + if (minId != null) { + pagingData = '$pagingData&min_id=$minId'; + } + + if (sinceId != null) { + pagingData = '$pagingData&since_id=$sinceId'; + } + + if (maxId != null) { + pagingData = '$pagingData&max_id=$maxId'; + } + + return pagingData; + } + + @override + String toString() { + return 'PagingData{maxId: $maxId, minId: $minId, sinceId: $sinceId, limit: $limit}'; + } + + @override + bool operator ==(Object other) => + identical(this, other) || + other is PagingData && + runtimeType == other.runtimeType && + minId == other.minId && + maxId == other.maxId && + sinceId == other.sinceId && + limit == other.limit; + + @override + int get hashCode => + minId.hashCode ^ maxId.hashCode ^ sinceId.hashCode ^ limit.hashCode; +} diff --git a/lib/services/connections_manager.dart b/lib/services/connections_manager.dart index 09e7472..e829af8 100644 --- a/lib/services/connections_manager.dart +++ b/lib/services/connections_manager.dart @@ -7,6 +7,7 @@ import 'package:result_monad/result_monad.dart'; import '../data/interfaces/connections_repo_intf.dart'; import '../data/interfaces/groups_repo.intf.dart'; +import '../friendica_client/paging_data.dart'; import '../globals.dart'; import '../models/connection.dart'; import '../models/exec_error.dart'; @@ -150,27 +151,29 @@ class ConnectionsManager extends ChangeNotifier { final results = {}; var moreResults = true; var maxId = -1; - const limit = 1000; + const limit = 200; + var currentPage = PagingData(limit: limit); while (moreResults) { - await client.getMyFollowers(sinceId: maxId, limit: limit).match( - onSuccess: (followers) { - for (final f in followers) { + await client.getMyFollowers(currentPage).match(onSuccess: (followers) { + for (final f in followers.data) { results[f.id] = f.copy(status: ConnectionStatus.theyFollowYou); int id = int.parse(f.id); maxId = max(maxId, id); } - moreResults = followers.length >= limit; + if (followers.next != null) { + currentPage = followers.next!; + } + moreResults = followers.next != null; }, onError: (error) { _logger.severe('Error getting followers data: $error'); }); } moreResults = true; - maxId = -1; + currentPage = PagingData(limit: limit); while (moreResults) { - await client.getMyFollowing(sinceId: maxId, limit: limit).match( - onSuccess: (following) { - for (final f in following) { + await client.getMyFollowing(currentPage).match(onSuccess: (following) { + for (final f in following.data) { if (results.containsKey(f.id)) { results[f.id] = f.copy(status: ConnectionStatus.mutual); } else { @@ -179,7 +182,10 @@ class ConnectionsManager extends ChangeNotifier { int id = int.parse(f.id); maxId = max(maxId, id); } - moreResults = following.length >= limit; + if (following.next != null) { + currentPage = following.next!; + } + moreResults = following.next != null; }, onError: (error) { _logger.severe('Error getting followers data: $error'); }); @@ -188,7 +194,7 @@ class ConnectionsManager extends ChangeNotifier { addAllConnections(results.values); final myContacts = conRepo.getMyContacts().toList(); myContacts.sort((c1, c2) => c1.name.compareTo(c2.name)); - _logger.finest('# Contacts:${myContacts.length}'); + _logger.fine('# Contacts:${myContacts.length}'); notifyListeners(); } diff --git a/lib/services/entry_manager_service.dart b/lib/services/entry_manager_service.dart index 4c341a7..a093cc5 100644 --- a/lib/services/entry_manager_service.dart +++ b/lib/services/entry_manager_service.dart @@ -4,6 +4,7 @@ import 'package:path/path.dart' as p; import 'package:result_monad/result_monad.dart'; import '../friendica_client/friendica_client.dart'; +import '../friendica_client/paging_data.dart'; import '../globals.dart'; import '../models/TimelineIdentifiers.dart'; import '../models/entry_tree_item.dart'; @@ -189,8 +190,13 @@ class EntryManagerService extends ChangeNotifier { } final client = clientResult.value; - final itemsResult = - await client.getTimeline(type: type, maxId: maxId, sinceId: sinceId); + final itemsResult = await client.getTimeline( + type: type, + page: PagingData( + maxId: maxId > 0 ? maxId : null, + sinceId: sinceId > 0 ? sinceId : null, + ), + ); if (itemsResult.isFailure) { _logger.severe('Error getting timeline: ${itemsResult.error}'); return itemsResult.errorCast(); diff --git a/test/paged_response_test.dart b/test/paged_response_test.dart new file mode 100644 index 0000000..ff4bfeb --- /dev/null +++ b/test/paged_response_test.dart @@ -0,0 +1,55 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:relatica/friendica_client/paged_response.dart'; +import 'package:relatica/friendica_client/paging_data.dart'; + +void main() { + const data = 'Hello'; + group('Test fromLinkHeader', () { + test('Null header (as if not there)', () { + expect( + PagedResponse.fromLinkHeader(null, data).value, + equals(PagedResponse(data)), + ); + }); + + test('Empty header', () { + expect( + PagedResponse.fromLinkHeader('', data).value, + equals(PagedResponse(data)), + ); + }); + + test('Not a previous/next header', () { + expect( + PagedResponse.fromLinkHeader( + '; rel="preconnect"', + data, + ).value, + equals(PagedResponse(data)), + ); + }); + + test('Previous and next', () { + expect( + PagedResponse.fromLinkHeader( + '; rel="next", ; rel="prev"', + data, + ).value, + equals(PagedResponse( + data, + previous: PagingData(minId: 590), + next: PagingData(maxId: 550), + )), + ); + }); + }); + + test('Test Mapping', () { + final original = PagedResponse(data, + previous: PagingData(minId: 2), next: PagingData(maxId: 3)); + expect( + original.map((data) => data.length), + equals(PagedResponse(data.length, + previous: PagingData(minId: 2), next: PagingData(maxId: 3)))); + }); +} diff --git a/test/paging_data_test.dart b/test/paging_data_test.dart new file mode 100644 index 0000000..8a31fb9 --- /dev/null +++ b/test/paging_data_test.dart @@ -0,0 +1,70 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:relatica/friendica_client/paging_data.dart'; + +void main() { + group('From Query Parameters', () { + test('No query string', () { + final paging = PagingData.fromQueryParameters( + Uri.parse('https://localhost'), + ); + expect(paging, equals(PagingData())); + }); + test('All Terms', () { + final paging = PagingData.fromQueryParameters( + Uri.parse( + 'https://localhost?&limit=49&max_id=48&min_id=46&since_id=47'), + ); + expect(paging, + equals(PagingData(maxId: 48, sinceId: 47, minId: 46, limit: 49))); + }); + }); + + group('To query parameters', () { + test('Default', () { + expect( + PagingData().toQueryParameters(), + equals('limit=50'), + ); + }); + + test('Specific limit only', () { + expect( + PagingData(limit: 10).toQueryParameters(), + equals('limit=10'), + ); + }); + + test('MinID only', () { + expect( + PagingData(maxId: 10).toQueryParameters(), + equals('limit=50&min_id=10'), + ); + }); + + test('MaxID only', () { + expect( + PagingData(maxId: 10).toQueryParameters(), + equals('limit=50&max_id=10'), + ); + }); + + test('SinceID only', () { + expect( + PagingData(sinceId: 10).toQueryParameters(), + equals('limit=50&since_id=10'), + ); + }); + + test('All Defined', () { + expect( + PagingData( + minId: 9, + sinceId: 10, + maxId: 11, + limit: 12, + ).toQueryParameters(), + equals('limit=12&min_id=9&since_id=10&max_id=11'), + ); + }); + }); +}