I'm trying to create a whitelabel streaming/cloud file management tool for mobile + Windows systems. The goal is to be able to view remote content and download it with the app serving as "proxy" for the files- only the app can open them, when downloading it encrypts & saves data locally. Currently I've tried Hive SQLite3 and right now I'm starting all over with Isar.
None of the above solutions work well with files over 2GB. Hive takes 7 minutes to save & encrypt 3GB data blob. What's worse saved file is over 9GB total. SQLite does a better job at 3.5 minutes while also decreasing final file size but that also seems like too long of a process.
Another thing is when encrypting & saving data processor, memory & disk usage are at 99% all the time on a Windows machine. I don't want to imagine how hard that process would hit a phone from like 2017.
try this code (it uses blowfish_ecb encryption algorithm but any other can be used as well), notice that when decrypting we dont have to wrap the original stream and use it directly (return (encrypt? makeBigChunksAndPadZeros(inStream, inStreamLength, outMap) : inStream))
class FooWidget1 extends StatefulWidget {
@override
State<FooWidget1> createState() => _FooWidget1State();
}
class _FooWidget1State extends State<FooWidget1> {
final notifier = ValueNotifier(0.0);
Duration timeSpan = Duration.zero;
Map<String, dynamic> out = {};
String status = '';
@override
Widget build(BuildContext context) {
return Align(
alignment: Alignment.topCenter,
child: Column(
children: [
ElevatedButton(
onPressed: () async {
final start = DateTime.now();
int length;
File inFile;
File outFile;
// case 1: encrypting from network
// final uri = Uri.parse('http://0.0.0.0:8000/open.mp4');
// final response = await http.Request('get', uri).send();
// // print(response.headers);
// length = response.contentLength!;
// outFile = File('secret.file');
// setState(() {
// status = 'encrypting\n\nsrc: [$uri]\ndst: [$outFile]\nlength: $length bytes';
// });
// out = await crypt(
// encrypt: true,
// key: 'foo bar key',
// inStream: response.stream,
// inStreamLength: length,
// outFile: outFile,
// notifier: notifier,
// );
// end of case 1
// case 2: encrypting from file
// inFile = File('open.mp4');
// length = await inFile.length();
// outFile = File('secret.file');
// setState(() {
// status = 'encrypting\n\nsrc: [$inFile]\ndst: [$outFile]\nlength: $length bytes';
// });
// out = await crypt(
// encrypt: true,
// key: 'foo bar key',
// inStream: inFile.openRead(),
// inStreamLength: length,
// outFile: outFile,
// notifier: notifier,
// );
// end of case 2
// case 3: decrypting from file
inFile = File('secret.file');
length = await inFile.length();
outFile = File('open1.mp4');
setState(() {
status = 'decrypting\n\nsrc: [$inFile]\ndst: [$outFile]\nlength: $length bytes';
});
out = await crypt(
encrypt: false,
key: 'foo bar key',
inStream: inFile.openRead(),
inStreamLength: length,
outFile: outFile,
notifier: notifier,
pad: 7,
);
// end of case 3
timeSpan = DateTime.now().difference(start);
setState(() {
status = status + '\n\ntook: ${timeSpan.inMilliseconds / 1000}s\npad: ${out['pad']}';
});
},
child: const Text('start crypting'),
),
SizedBox.fromSize(
size: const Size.square(100),
child: Padding(
padding: const EdgeInsets.all(8),
child: AnimatedBuilder(
animation: notifier,
builder: (ctx, child) => CircularProgressIndicator(value: notifier.value),
),
),
),
Text(status, textScaleFactor: 1.5),
],
),
);
}
Future<Map<String, dynamic>> crypt({
bool encrypt = true,
required String key,
required Stream<List<int>> inStream,
required int inStreamLength,
required File outFile,
ValueNotifier<double>? notifier,
int pad = 0,
}) async {
int readBytes = 0;
double oldValue = 0;
// it needs: import 'package:blowfish_ecb/blowfish_ecb.dart';
final blowfishECB = BlowfishECB(Uint8List.fromList(utf8.encode(key)));
List<int> mapper(List<int> data) {
if (notifier != null && inStreamLength > 0) {
readBytes += data.length;
final currentValue = readBytes / inStreamLength;
if (currentValue > oldValue + 0.01) {
oldValue = currentValue;
notifier.value = currentValue;
}
}
if (encrypt) {
data = blowfishECB.encode(data);
} else {
data = blowfishECB.decode(data);
if (readBytes == inStreamLength && pad != 0) {
// the last chunk with non zero [pad]
print('crypt: removing $pad byte(s)');
data = data.sublist(0, data.length - pad);
}
}
return data;
}
final outMap = {
'size': 0,
'pad': 0,
};
return (encrypt? makeBigChunksAndPadZeros(inStream, inStreamLength, outMap) : inStream)
.map(mapper)
.pipe(outFile.openWrite())
.then((value) => outMap..['size'] = readBytes);
}
Stream<List<int>> makeBigChunksAndPadZeros(Stream<List<int>> inStream, int inStreamLength, Map outMap) async* {
// TODO make it bigger / smaller
const chunkSize = 1 * 1024 * 1024;
int index = 0;
final reader = ChunkedStreamReader(inStream);
while (true) {
var chunk = await reader.readChunk(chunkSize);
index += chunk.length;
if (index != inStreamLength) {
// normal chunk
yield chunk;
} else {
// the last chunk
if (chunk.isNotEmpty) {
// not empty chunk: add zeroes if needed
final pad = 8 - chunk.length % 8;
if (pad != 8) {
print('makeBigChunksAndPadZeros: adding $pad zero(s)');
chunk = List.of(chunk.followedBy(List.filled(pad, 0)));
outMap['pad'] = pad;
}
yield chunk;
}
break;
}
}
}
}