Add initial paging architecture but only use paging responses on getting followers/following table

codemagic-setup
Hank Grabowski 2023-01-23 22:37:09 -05:00
rodzic bf13e0674b
commit 49864d4f97
7 zmienionych plików z 369 dodań i 77 usunięć

Wyświetl plik

@ -6,6 +6,7 @@ import 'package:http_parser/http_parser.dart';
import 'package:logging/logging.dart';
import 'package:result_monad/result_monad.dart';
import '../friendica_client/paged_response.dart';
import '../globals.dart';
import '../models/TimelineIdentifiers.dart';
import '../models/connection.dart';
@ -25,6 +26,7 @@ import '../serializers/mastodon/group_data_mastodon_extensions.dart';
import '../serializers/mastodon/notification_mastodon_extension.dart';
import '../serializers/mastodon/timeline_entry_mastodon_extensions.dart';
import '../services/auth_service.dart';
import 'paging_data.dart';
class FriendicaClient {
static final _logger = Logger('$FriendicaClient');
@ -43,13 +45,14 @@ class FriendicaClient {
_authHeader = "Basic $encodedAuthString";
}
// TODO Convert Notifications to using paging for real
FutureResult<List<UserNotification>, ExecError> getNotifications() async {
final url =
'https://$serverName/api/v1/notifications?include_all=true&limit=200';
final request = Uri.parse(url);
_logger.finest(() => 'Getting new notifications');
return (await _getApiListRequest(request).andThenSuccessAsync(
(notificationsJson) async => notificationsJson
(notificationsJson) async => notificationsJson.data
.map((json) => NotificationMastodonExtension.fromJson(json))
.toList()))
.mapError((error) {
@ -79,12 +82,13 @@ class FriendicaClient {
return response.mapValue((value) => true);
}
// TODO Convert Albums to using paging for real
FutureResult<List<GalleryData>, ExecError> getGalleryData() async {
_logger.finest(() => 'Getting gallery data');
final url = 'https://$serverName/api/friendica/photoalbums';
final request = Uri.parse(url);
return (await _getApiListRequest(request).andThenSuccessAsync(
(albumsJson) async => albumsJson
(albumsJson) async => albumsJson.data
.map((json) => GalleryDataFriendicaExtensions.fromJson(json))
.toList()))
.mapError((error) => error is ExecError
@ -92,6 +96,7 @@ class FriendicaClient {
: ExecError(type: ErrorType.localError, message: error.toString()));
}
// TODO Convert Gallery Images to using paging for real
FutureResult<List<ImageEntry>, ExecError> getGalleryImages(
String galleryName) async {
_logger.finest(() => 'Getting gallery data');
@ -99,7 +104,7 @@ class FriendicaClient {
'https://$serverName/api/friendica/photoalbum?album=$galleryName';
final request = Uri.parse(url);
return (await _getApiListRequest(request).andThenSuccessAsync(
(imagesJson) async => imagesJson
(imagesJson) async => imagesJson.data
.map((json) => ImageEntryFriendicaExtension.fromJson(json))
.toList()))
.mapError((error) => error is ExecError
@ -107,12 +112,13 @@ class FriendicaClient {
: ExecError(type: ErrorType.localError, message: error.toString()));
}
// TODO Convert Groups to using paging for real (if it is supported)
FutureResult<List<GroupData>, ExecError> getGroups() async {
_logger.finest(() => 'Getting group (Mastodon List) data');
final url = 'https://$serverName/api/v1/lists';
final request = Uri.parse(url);
return (await _getApiListRequest(request).andThenSuccessAsync(
(listsJson) async => listsJson
(listsJson) async => listsJson.data
.map((json) => GroupDataMastodonExtensions.fromJson(json))
.toList()))
.mapError((error) => error is ExecError
@ -146,33 +152,34 @@ class FriendicaClient {
return (await _deleteUrl(request, requestData)).mapValue((_) => true);
}
FutureResult<List<Connection>, ExecError> getMyFollowing(
{int sinceId = -1, int maxId = -1, int limit = 50}) async {
_logger.finest(() =>
'Getting following data since $sinceId, maxId $maxId, limit $limit');
FutureResult<PagedResponse<List<Connection>>, ExecError> getMyFollowing(
PagingData page) async {
_logger.finest(() => 'Getting following with paging data $page');
final myId = getIt<AuthService>().currentId;
final paging =
_buildPagingData(sinceId: sinceId, maxId: maxId, limit: limit);
final baseUrl = 'https://$serverName/api/v1/accounts/$myId';
return (await _getApiListRequest(Uri.parse('$baseUrl/following&$paging'))
.andThenSuccessAsync((listJson) async => listJson
.map((json) => ConnectionMastodonExtensions.fromJson(json))
.toList()))
final result = await _getApiListRequest(
Uri.parse('$baseUrl/following?${page.toQueryParameters()}'),
);
return result
.andThenSuccess((response) => response.map((jsonArray) => jsonArray
.map((json) => ConnectionMastodonExtensions.fromJson(json))
.toList()))
.execErrorCast();
}
FutureResult<List<Connection>, ExecError> getMyFollowers(
{int sinceId = -1, int maxId = -1, int limit = 50}) async {
_logger.finest(() =>
'Getting followers data since $sinceId, maxId $maxId, limit $limit');
FutureResult<PagedResponse<List<Connection>>, ExecError> getMyFollowers(
PagingData page) async {
_logger.finest(() => 'Getting followers data with page data $page');
final myId = getIt<AuthService>().currentId;
final paging =
_buildPagingData(sinceId: sinceId, maxId: maxId, limit: limit);
final baseUrl = 'https://$serverName/api/v1/accounts/$myId';
return (await _getApiListRequest(Uri.parse('$baseUrl/followers&$paging'))
.andThenSuccessAsync((listJson) async => listJson
.map((json) => ConnectionMastodonExtensions.fromJson(json))
.toList()))
final result1 = await _getApiListRequest(
Uri.parse('$baseUrl/followers&${page.toQueryParameters()}'),
);
return result1
.andThenSuccess((response) => response.map((jsonArray) => jsonArray
.map((json) => ConnectionMastodonExtensions.fromJson(json))
.toList()))
.execErrorCast();
}
@ -185,14 +192,14 @@ class FriendicaClient {
final baseUrl = 'https://$serverName/api/v1/accounts/$myId';
final following =
await _getApiListRequest(Uri.parse('$baseUrl/following$paging')).fold(
onSuccess: (followings) => followings.isNotEmpty,
onSuccess: (followings) => followings.data.isNotEmpty,
onError: (error) {
_logger.severe('Error getting following list: $error');
return false;
});
final follower =
await _getApiListRequest(Uri.parse('$baseUrl/followers$paging')).fold(
onSuccess: (followings) => followings.isNotEmpty,
onSuccess: (followings) => followings.data.isNotEmpty,
onError: (error) {
_logger.severe('Error getting follower list: $error');
return false;
@ -209,6 +216,7 @@ class FriendicaClient {
return Result.ok(connection.copy(status: status));
}
// TODO Convert groups for connection to using paging for real (if available)
FutureResult<List<GroupData>, ExecError> getMemberGroupsForConnection(
String connectionId) async {
_logger.finest(() =>
@ -216,12 +224,13 @@ class FriendicaClient {
final url = 'https://$serverName/api/v1/accounts/$connectionId/lists';
final request = Uri.parse(url);
return (await _getApiListRequest(request).andThenSuccessAsync(
(listsJson) async => listsJson
(listsJson) async => listsJson.data
.map((json) => GroupDataMastodonExtensions.fromJson(json))
.toList()))
.mapError((error) => error as ExecError);
}
// TODO Convert User Timeline to using paging for real
FutureResult<List<TimelineEntry>, ExecError> getUserTimeline(
{String userId = '', int page = 1, int count = 10}) async {
_logger.finest(() => 'Getting user timeline for $userId');
@ -232,29 +241,23 @@ class FriendicaClient {
: '${baseUrl}screen_name=$userId$pagingData';
final request = Uri.parse(url);
return (await _getApiListRequest(request).andThenSuccessAsync(
(postsJson) async => postsJson
(postsJson) async => postsJson.data
.map((json) => TimelineEntryMastodonExtensions.fromJson(json))
.toList()))
.mapError((error) => error as ExecError);
}
FutureResult<List<TimelineEntry>, ExecError> getTimeline(
{required TimelineIdentifiers type,
int sinceId = 0,
int maxId = 0,
int limit = 20}) async {
{required TimelineIdentifiers type, required PagingData page}) async {
final String timelinePath = _typeToTimelinePath(type);
final String timelineQPs = _typeToTimelineQueryParameters(type);
final baseUrl = 'https://$serverName/api/v1/$timelinePath';
final pagingData =
_buildPagingData(sinceId: sinceId, maxId: maxId, limit: limit);
final url = '$baseUrl?exclude_replies=true&$pagingData&$timelineQPs';
final url =
'$baseUrl?exclude_replies=true&${page.toQueryParameters()}&$timelineQPs';
final request = Uri.parse(url);
_logger.finest(() =>
'Getting ${type.toHumanKey()} limit $limit sinceId: $sinceId maxId: $maxId : $url');
_logger.finest(() => 'Getting ${type.toHumanKey()} with paging data $page');
return (await _getApiListRequest(request).andThenSuccessAsync(
(postsJson) async => postsJson
(postsJson) async => postsJson.data
.map((json) => TimelineEntryMastodonExtensions.fromJson(json))
.toList()))
.execErrorCast();
@ -282,6 +285,7 @@ class FriendicaClient {
}
}
// TODO Convert getPostOrComment to using paging for real
FutureResult<List<TimelineEntry>, ExecError> getPostOrComment(String id,
{bool fullContext = false}) async {
return (await runCatchingAsync(() async {
@ -306,10 +310,7 @@ class FriendicaClient {
}
}));
}))
.mapError((error) => ExecError(
type: ErrorType.parsingError,
message: error.toString(),
));
.execErrorCastAsync();
}
FutureResult<bool, ExecError> deleteEntryById(String id) async {
@ -515,7 +516,7 @@ class FriendicaClient {
return Result.ok(newImageData);
}
FutureResult<String, ExecError> _getUrl(Uri url) async {
FutureResult<PagedResponse<String>, ExecError> _getUrl(Uri url) async {
_logger.finer('GET: $url');
try {
final response = await http.get(
@ -531,7 +532,10 @@ class FriendicaClient {
type: ErrorType.authentication,
message: '${response.statusCode}: ${response.reasonPhrase}'));
}
return Result.ok(utf8.decode(response.bodyBytes));
return PagedResponse.fromLinkHeader(
response.headers['link'],
utf8.decode(response.bodyBytes),
);
} catch (e) {
return Result.error(
ExecError(type: ErrorType.localError, message: e.toString()));
@ -589,16 +593,20 @@ class FriendicaClient {
}
}
FutureResult<List<dynamic>, ExecError> _getApiListRequest(Uri url) async {
FutureResult<PagedResponse<List<dynamic>>, ExecError> _getApiListRequest(
Uri url) async {
return (await _getUrl(url).andThenSuccessAsync(
(jsonText) async => jsonDecode(jsonText) as List<dynamic>))
(response) async =>
response.map((data) => jsonDecode(data) as List<dynamic>),
))
.mapError((error) => error as ExecError);
}
FutureResult<dynamic, ExecError> _getApiRequest(Uri url) async {
return (await _getUrl(url)
.andThenSuccessAsync((jsonText) async => jsonDecode(jsonText)))
.mapError((error) => error as ExecError);
return (await _getUrl(url).andThenSuccessAsync(
(response) async => jsonDecode(response.data),
))
.execErrorCastAsync();
}
String _typeToTimelinePath(TimelineIdentifiers type) {
@ -632,20 +640,6 @@ class FriendicaClient {
}
}
String _buildPagingData(
{required int sinceId, required int maxId, required int limit}) {
var pagingData = 'limit=$limit';
if (maxId > 0) {
pagingData = '$pagingData&max_id=$maxId';
}
if (sinceId > 0) {
pagingData = '&since_id=$sinceId';
}
return pagingData;
}
Connection _updateConnectionFromFollowRequestResult(
Connection connection, String jsonString) {
final json = jsonDecode(jsonString) as Map<String, dynamic>;

Wyświetl plik

@ -0,0 +1,97 @@
import 'package:logging/logging.dart';
import 'package:result_monad/result_monad.dart';
import '../models/exec_error.dart';
import 'paging_data.dart';
final _logger = Logger('PagedResponse');
class PagedResponse<T> {
PagingData? previous;
PagingData? next;
T data;
PagedResponse(this.data, {this.previous, this.next});
bool get hasMorePages => previous != null || next != null;
static Result<PagedResponse<T>, ExecError> fromLinkHeader<T>(
String? linkHeader, T data) {
if (linkHeader == null || linkHeader.isEmpty) {
return Result.ok(PagedResponse(data));
}
String? previousPage;
String? nextPage;
for (String linkTerms in linkHeader.trim().split(',')) {
if (linkHeader.isEmpty) {
return buildErrorResult(
type: ErrorType.parsingError,
message: 'Link header element is blank',
);
}
final paging = linkTerms.split(';');
if (paging.length != 2) {
return buildErrorResult(
type: ErrorType.parsingError,
message:
'Incorrect number of elements, ${paging.length} != 2, for: $linkTerms',
);
}
final urlPieceString = paging.first.trim();
if (!urlPieceString.startsWith('<') && !urlPieceString.endsWith('>')) {
return buildErrorResult(
type: ErrorType.parsingError,
message:
'Link URL is malformed (no leading trailing <>): $urlPieceString',
);
}
final url = urlPieceString.substring(1, urlPieceString.length - 1);
final directionString = paging.last.trim();
if (directionString == 'rel="prev"') {
previousPage = url;
} else if (directionString == 'rel="next"') {
nextPage = url;
} else {
_logger.info('Unknown paging data: $directionString for url: $url');
}
}
return Result.ok(PagedResponse(
data,
previous: previousPage == null
? null
: PagingData.fromQueryParameters(
Uri.parse(previousPage),
),
next: nextPage == null
? null
: PagingData.fromQueryParameters(
Uri.parse(nextPage),
),
));
}
PagedResponse<T2> map<T2>(T2 Function(T data) func) => PagedResponse(
func(data),
previous: previous,
next: next,
);
@override
String toString() {
return 'PagedResponse{previous: $previous, next: $next, data: $data}';
}
@override
bool operator ==(Object other) =>
identical(this, other) ||
other is PagedResponse &&
runtimeType == other.runtimeType &&
previous == other.previous &&
next == other.next &&
data == other.data;
@override
int get hashCode => previous.hashCode ^ next.hashCode ^ data.hashCode;
}

Wyświetl plik

@ -0,0 +1,64 @@
class PagingData {
static const DEFAULT_LIMIT = 50;
final int? minId;
final int? maxId;
final int? sinceId;
final int limit;
PagingData({
this.minId,
this.maxId,
this.sinceId,
this.limit = DEFAULT_LIMIT,
});
factory PagingData.fromQueryParameters(Uri uri) {
final minIdString = uri.queryParameters['min_id'];
final maxIdString = uri.queryParameters['max_id'];
final sinceIdString = uri.queryParameters['since_id'];
final limitString = uri.queryParameters['limit'];
return PagingData(
minId: int.tryParse(minIdString ?? ''),
maxId: int.tryParse(maxIdString ?? ''),
sinceId: int.tryParse(sinceIdString ?? ''),
limit: int.tryParse(limitString ?? '') ?? DEFAULT_LIMIT,
);
}
String toQueryParameters() {
var pagingData = 'limit=$limit';
if (minId != null) {
pagingData = '$pagingData&min_id=$minId';
}
if (sinceId != null) {
pagingData = '$pagingData&since_id=$sinceId';
}
if (maxId != null) {
pagingData = '$pagingData&max_id=$maxId';
}
return pagingData;
}
@override
String toString() {
return 'PagingData{maxId: $maxId, minId: $minId, sinceId: $sinceId, limit: $limit}';
}
@override
bool operator ==(Object other) =>
identical(this, other) ||
other is PagingData &&
runtimeType == other.runtimeType &&
minId == other.minId &&
maxId == other.maxId &&
sinceId == other.sinceId &&
limit == other.limit;
@override
int get hashCode =>
minId.hashCode ^ maxId.hashCode ^ sinceId.hashCode ^ limit.hashCode;
}

Wyświetl plik

@ -7,6 +7,7 @@ import 'package:result_monad/result_monad.dart';
import '../data/interfaces/connections_repo_intf.dart';
import '../data/interfaces/groups_repo.intf.dart';
import '../friendica_client/paging_data.dart';
import '../globals.dart';
import '../models/connection.dart';
import '../models/exec_error.dart';
@ -150,27 +151,29 @@ class ConnectionsManager extends ChangeNotifier {
final results = <String, Connection>{};
var moreResults = true;
var maxId = -1;
const limit = 1000;
const limit = 200;
var currentPage = PagingData(limit: limit);
while (moreResults) {
await client.getMyFollowers(sinceId: maxId, limit: limit).match(
onSuccess: (followers) {
for (final f in followers) {
await client.getMyFollowers(currentPage).match(onSuccess: (followers) {
for (final f in followers.data) {
results[f.id] = f.copy(status: ConnectionStatus.theyFollowYou);
int id = int.parse(f.id);
maxId = max(maxId, id);
}
moreResults = followers.length >= limit;
if (followers.next != null) {
currentPage = followers.next!;
}
moreResults = followers.next != null;
}, onError: (error) {
_logger.severe('Error getting followers data: $error');
});
}
moreResults = true;
maxId = -1;
currentPage = PagingData(limit: limit);
while (moreResults) {
await client.getMyFollowing(sinceId: maxId, limit: limit).match(
onSuccess: (following) {
for (final f in following) {
await client.getMyFollowing(currentPage).match(onSuccess: (following) {
for (final f in following.data) {
if (results.containsKey(f.id)) {
results[f.id] = f.copy(status: ConnectionStatus.mutual);
} else {
@ -179,7 +182,10 @@ class ConnectionsManager extends ChangeNotifier {
int id = int.parse(f.id);
maxId = max(maxId, id);
}
moreResults = following.length >= limit;
if (following.next != null) {
currentPage = following.next!;
}
moreResults = following.next != null;
}, onError: (error) {
_logger.severe('Error getting followers data: $error');
});
@ -188,7 +194,7 @@ class ConnectionsManager extends ChangeNotifier {
addAllConnections(results.values);
final myContacts = conRepo.getMyContacts().toList();
myContacts.sort((c1, c2) => c1.name.compareTo(c2.name));
_logger.finest('# Contacts:${myContacts.length}');
_logger.fine('# Contacts:${myContacts.length}');
notifyListeners();
}

Wyświetl plik

@ -4,6 +4,7 @@ import 'package:path/path.dart' as p;
import 'package:result_monad/result_monad.dart';
import '../friendica_client/friendica_client.dart';
import '../friendica_client/paging_data.dart';
import '../globals.dart';
import '../models/TimelineIdentifiers.dart';
import '../models/entry_tree_item.dart';
@ -189,8 +190,13 @@ class EntryManagerService extends ChangeNotifier {
}
final client = clientResult.value;
final itemsResult =
await client.getTimeline(type: type, maxId: maxId, sinceId: sinceId);
final itemsResult = await client.getTimeline(
type: type,
page: PagingData(
maxId: maxId > 0 ? maxId : null,
sinceId: sinceId > 0 ? sinceId : null,
),
);
if (itemsResult.isFailure) {
_logger.severe('Error getting timeline: ${itemsResult.error}');
return itemsResult.errorCast();

Wyświetl plik

@ -0,0 +1,55 @@
import 'package:flutter_test/flutter_test.dart';
import 'package:relatica/friendica_client/paged_response.dart';
import 'package:relatica/friendica_client/paging_data.dart';
void main() {
const data = 'Hello';
group('Test fromLinkHeader', () {
test('Null header (as if not there)', () {
expect(
PagedResponse.fromLinkHeader(null, data).value,
equals(PagedResponse(data)),
);
});
test('Empty header', () {
expect(
PagedResponse.fromLinkHeader('', data).value,
equals(PagedResponse(data)),
);
});
test('Not a previous/next header', () {
expect(
PagedResponse.fromLinkHeader(
'<https://example.com>; rel="preconnect"',
data,
).value,
equals(PagedResponse(data)),
);
});
test('Previous and next', () {
expect(
PagedResponse.fromLinkHeader(
'<https://friendica.myportal.social/api/v1/accounts/1/followers?max_id=550>; rel="next", <https://friendica.myportal.social/api/v1/accounts/1/followers?min_id=590>; rel="prev"',
data,
).value,
equals(PagedResponse(
data,
previous: PagingData(minId: 590),
next: PagingData(maxId: 550),
)),
);
});
});
test('Test Mapping', () {
final original = PagedResponse(data,
previous: PagingData(minId: 2), next: PagingData(maxId: 3));
expect(
original.map((data) => data.length),
equals(PagedResponse(data.length,
previous: PagingData(minId: 2), next: PagingData(maxId: 3))));
});
}

Wyświetl plik

@ -0,0 +1,70 @@
import 'package:flutter_test/flutter_test.dart';
import 'package:relatica/friendica_client/paging_data.dart';
void main() {
group('From Query Parameters', () {
test('No query string', () {
final paging = PagingData.fromQueryParameters(
Uri.parse('https://localhost'),
);
expect(paging, equals(PagingData()));
});
test('All Terms', () {
final paging = PagingData.fromQueryParameters(
Uri.parse(
'https://localhost?&limit=49&max_id=48&min_id=46&since_id=47'),
);
expect(paging,
equals(PagingData(maxId: 48, sinceId: 47, minId: 46, limit: 49)));
});
});
group('To query parameters', () {
test('Default', () {
expect(
PagingData().toQueryParameters(),
equals('limit=50'),
);
});
test('Specific limit only', () {
expect(
PagingData(limit: 10).toQueryParameters(),
equals('limit=10'),
);
});
test('MinID only', () {
expect(
PagingData(maxId: 10).toQueryParameters(),
equals('limit=50&min_id=10'),
);
});
test('MaxID only', () {
expect(
PagingData(maxId: 10).toQueryParameters(),
equals('limit=50&max_id=10'),
);
});
test('SinceID only', () {
expect(
PagingData(sinceId: 10).toQueryParameters(),
equals('limit=50&since_id=10'),
);
});
test('All Defined', () {
expect(
PagingData(
minId: 9,
sinceId: 10,
maxId: 11,
limit: 12,
).toQueryParameters(),
equals('limit=12&min_id=9&since_id=10&max_id=11'),
);
});
});
}