I currently have a dilemma of scraping a website for data (specifically mooncalc.org), I believe, and this is a very weak belief, that I've got the data and am assessing the right part.
Resources: https://itnext.io/write-your-first-web-scraper-in-dart-243c7bb4d05 https://www.mooncalc.org/#/48.8583,-71.3292,10/2019.12.25/20:10/1/0 through developer tools (I'm specifically trying to get the moon age but I want this problem to be solved not with specifics)
(other: I've found it very annoying that mooncalc states that it has an API, but there link just leads to a website with HTML So, not to bother them if I can freely get data from there website.)
I've been trying to get the HTML and parse it with HTPL and HTML flutter dependencies.
https://pub.dev/packages/http https://pub.dev/packages/html#-changelog-tab-
int process = 0;
LocationData _location;
double _lunarDay;
DateTime _nextEkadashi;
@override
void initState() {
super.initState();
if (process == 0) {
getLocation();
// initiate();
}
}
Future initiate() async {
var client = http.Client();
http.Response response = await client.get(
'https://mooncalc.org/#/${_location.latitude},${_location.longitude},null/null/null/null/null');
ok.Document document = parse(response.body);
List<ok.Element> maybe1 = document.getElementsByClassName("moontext alter");
print(maybe1[0].text);
}
Do imagine that the location data, now this returns ..., without the .text in print, it returns HTML span> (or something close as this editor actually makes an HTML span), how can I read that data through nifty flutter or command line. Also, there's only one item in the List
Also I prefer flutter run to F5.
import 'package:flutter/material.dart';
import 'package:flutter/services.dart';
import 'package:html/dom.dart' as ok;
import 'dart:convert';
import 'package:location/location.dart';
import 'package:http/http.dart' as http;
import 'package:intl/intl.dart';
import 'package:html/parser.dart';
These are my imports, there are multiple conflicts so I had to rename them.
Pranits-iMac:Ekadashi pranitshah$ flutter analyze Analyzing Ekadashi...
No issues found! (ran in 332.3s) Pranits-iMac:Ekadashi pranitshah$ flutter doctor -v [✓] Flutter (Channel master, v1.13.6-pre.22, on Mac OS X 10.13.6 17G9016, locale en-US) • Flutter version 1.13.6-pre.22 at /Users/pranitshah/Developer/flutter • Framework revision d874596e38 (2 days ago), 2019-12-23 16:16:43 -0800 • Engine revision 33813929e3 • Dart version 2.8.0 (build 2.8.0-dev.0.0 886615d0f9)
[✓] Android toolchain - develop for Android devices (Android SDK version 29.0.2) • Android SDK at /Users/pranitshah/Library/Android/sdk • Android NDK location not configured (optional; useful for native profiling support) • Platform android-29, build-tools 29.0.2 • Java binary at: /Applications/Android Studio.app/Contents/jre/jdk/Contents/Home/bin/java • Java version OpenJDK Runtime Environment (build 1.8.0_152-release-1343-b01) • All Android licenses accepted.
[✓] Xcode - develop for iOS and macOS (Xcode 10.2.1) • Xcode at /Applications/Xcode10.2.1.app/Contents/Developer • Xcode 10.2.1, Build version 10E1001 • CocoaPods version 1.7.5
[✓] Chrome - develop for the web • Chrome at /Applications/Google Chrome.app/Contents/MacOS/Google Chrome
[✓] Android Studio (version 3.4) • Android Studio at /Applications/Android Studio.app/Contents • Flutter plugin version 38.2.1 • Dart plugin version 183.6270 • Java version OpenJDK Runtime Environment (build 1.8.0_152-release-1343-b01)
[✓] VS Code (version 1.41.1) • VS Code at /Applications/Visual Studio Code.app/Contents • Flutter extension version 3.7.1
[✓] Connected device (3 available) • SAMSUNG SM J120A • 42007cb0e416a3bd • android-arm • Android 6.0.1 (API 23) • Chrome • chrome • web-javascript • Google Chrome 79.0.3945.88 • Web Server • web-server • web-javascript • Flutter Tools
• No issues found!
I tried converting HTML to JSON for mooncalc.org , I got this output.
class HtmltoModel {
Head head;
List<Null> headers;
List<Null> jsonLd;
List<Forms> forms;
List<Tables> tables;
List<Null> iframes;
List<Null> embeds;
List<Imgs> imgs;
List<Links> links;
String content;
Null datetime;
HtmltoModel({this.head, this.headers, this.jsonLd, this.forms, this.tables, this.iframes, this.embeds, this.imgs, this.links, this.content, this.datetime});
HtmltoModel.fromJson(Map<String, dynamic> json) {
head = json['head'] != null ? new Head.fromJson(json['head']) : null;
if (json['headers'] != null) {
headers = new List<Null>();
json['headers'].forEach((v) { headers.add(new Null.fromJson(v)); });
}
if (json['jsonLd'] != null) {
jsonLd = new List<Null>();
json['jsonLd'].forEach((v) { jsonLd.add(new Null.fromJson(v)); });
}
if (json['forms'] != null) {
forms = new List<Forms>();
json['forms'].forEach((v) { forms.add(new Forms.fromJson(v)); });
}
if (json['tables'] != null) {
tables = new List<Tables>();
json['tables'].forEach((v) { tables.add(new Tables.fromJson(v)); });
}
if (json['iframes'] != null) {
iframes = new List<Null>();
json['iframes'].forEach((v) { iframes.add(new Null.fromJson(v)); });
}
if (json['embeds'] != null) {
embeds = new List<Null>();
json['embeds'].forEach((v) { embeds.add(new Null.fromJson(v)); });
}
if (json['imgs'] != null) {
imgs = new List<Imgs>();
json['imgs'].forEach((v) { imgs.add(new Imgs.fromJson(v)); });
}
if (json['links'] != null) {
links = new List<Links>();
json['links'].forEach((v) { links.add(new Links.fromJson(v)); });
}
content = json['content'];
datetime = json['datetime'];
}
Map<String, dynamic> toJson() {
final Map<String, dynamic> data = new Map<String, dynamic>();
if (this.head != null) {
data['head'] = this.head.toJson();
}
if (this.headers != null) {
data['headers'] = this.headers.map((v) => v.toJson()).toList();
}
if (this.jsonLd != null) {
data['jsonLd'] = this.jsonLd.map((v) => v.toJson()).toList();
}
if (this.forms != null) {
data['forms'] = this.forms.map((v) => v.toJson()).toList();
}
if (this.tables != null) {
data['tables'] = this.tables.map((v) => v.toJson()).toList();
}
if (this.iframes != null) {
data['iframes'] = this.iframes.map((v) => v.toJson()).toList();
}
if (this.embeds != null) {
data['embeds'] = this.embeds.map((v) => v.toJson()).toList();
}
if (this.imgs != null) {
data['imgs'] = this.imgs.map((v) => v.toJson()).toList();
}
if (this.links != null) {
data['links'] = this.links.map((v) => v.toJson()).toList();
}
data['content'] = this.content;
data['datetime'] = this.datetime;
return data;
}
}
class Head {
String title;
List<Meta> meta;
List<Link> link;
List<Script> script;
Head({this.title, this.meta, this.link, this.script});
Head.fromJson(Map<String, dynamic> json) {
title = json['title'];
if (json['meta'] != null) {
meta = new List<Meta>();
json['meta'].forEach((v) { meta.add(new Meta.fromJson(v)); });
}
if (json['link'] != null) {
link = new List<Link>();
json['link'].forEach((v) { link.add(new Link.fromJson(v)); });
}
if (json['script'] != null) {
script = new List<Script>();
json['script'].forEach((v) { script.add(new Script.fromJson(v)); });
}
}
Map<String, dynamic> toJson() {
final Map<String, dynamic> data = new Map<String, dynamic>();
data['title'] = this.title;
if (this.meta != null) {
data['meta'] = this.meta.map((v) => v.toJson()).toList();
}
if (this.link != null) {
data['link'] = this.link.map((v) => v.toJson()).toList();
}
if (this.script != null) {
data['script'] = this.script.map((v) => v.toJson()).toList();
}
return data;
}
}
class Meta {
String charset;
String name;
String content;
String property;
Meta({this.charset, this.name, this.content, this.property});
Meta.fromJson(Map<String, dynamic> json) {
charset = json['charset'];
name = json['name'];
content = json['content'];
property = json['property'];
}
Map<String, dynamic> toJson() {
final Map<String, dynamic> data = new Map<String, dynamic>();
data['charset'] = this.charset;
data['name'] = this.name;
data['content'] = this.content;
data['property'] = this.property;
return data;
}
}
class Link {
String rel;
String href;
String sizes;
String type;
String media;
Link({this.rel, this.href, this.sizes, this.type, this.media});
Link.fromJson(Map<String, dynamic> json) {
rel = json['rel'];
href = json['href'];
sizes = json['sizes'];
type = json['type'];
media = json['media'];
}
Map<String, dynamic> toJson() {
final Map<String, dynamic> data = new Map<String, dynamic>();
data['rel'] = this.rel;
data['href'] = this.href;
data['sizes'] = this.sizes;
data['type'] = this.type;
data['media'] = this.media;
return data;
}
}
class Script {
String src;
String type;
String async;
String id;
String dataBg;
String dataFg;
String dataLink;
String dataLinkmsg;
String dataLinktarget;
String dataEffect;
String dataCloseText;
String dataMoreinfo;
String dataCookie;
String dataTextAlign;
String dataFontSize;
String dataDivlinkbg;
String dataDivlink;
String dataZindex;
String dataMaxage;
String dataOpacity;
Script({this.src, this.type, this.async, this.id, this.dataBg, this.dataFg, this.dataLink, this.dataLinkmsg, this.dataLinktarget, this.dataEffect, this.dataCloseText, this.dataMoreinfo, this.dataCookie, this.dataTextAlign, this.dataFontSize, this.dataDivlinkbg, this.dataDivlink, this.dataZindex, this.dataMaxage, this.dataOpacity});
Script.fromJson(Map<String, dynamic> json) {
src = json['src'];
type = json['type'];
async = json['async'];
id = json['id'];
dataBg = json['data-bg'];
dataFg = json['data-fg'];
dataLink = json['data-link'];
dataLinkmsg = json['data-linkmsg'];
dataLinktarget = json['data-linktarget'];
dataEffect = json['data-effect'];
dataCloseText = json['data-close-text'];
dataMoreinfo = json['data-moreinfo'];
dataCookie = json['data-cookie'];
dataTextAlign = json['data-text-align'];
dataFontSize = json['data-font-size'];
dataDivlinkbg = json['data-divlinkbg'];
dataDivlink = json['data-divlink'];
dataZindex = json['data-zindex'];
dataMaxage = json['data-maxage'];
dataOpacity = json['data-opacity'];
}
Map<String, dynamic> toJson() {
final Map<String, dynamic> data = new Map<String, dynamic>();
data['src'] = this.src;
data['type'] = this.type;
data['async'] = this.async;
data['id'] = this.id;
data['data-bg'] = this.dataBg;
data['data-fg'] = this.dataFg;
data['data-link'] = this.dataLink;
data['data-linkmsg'] = this.dataLinkmsg;
data['data-linktarget'] = this.dataLinktarget;
data['data-effect'] = this.dataEffect;
data['data-close-text'] = this.dataCloseText;
data['data-moreinfo'] = this.dataMoreinfo;
data['data-cookie'] = this.dataCookie;
data['data-text-align'] = this.dataTextAlign;
data['data-font-size'] = this.dataFontSize;
data['data-divlinkbg'] = this.dataDivlinkbg;
data['data-divlink'] = this.dataDivlink;
data['data-zindex'] = this.dataZindex;
data['data-maxage'] = this.dataMaxage;
data['data-opacity'] = this.dataOpacity;
return data;
}
}
class Forms {
Attr attr;
List<Fields> fields;
Forms({this.attr, this.fields});
Forms.fromJson(Map<String, dynamic> json) {
attr = json['attr'] != null ? new Attr.fromJson(json['attr']) : null;
if (json['fields'] != null) {
fields = new List<Fields>();
json['fields'].forEach((v) { fields.add(new Fields.fromJson(v)); });
}
}
Map<String, dynamic> toJson() {
final Map<String, dynamic> data = new Map<String, dynamic>();
if (this.attr != null) {
data['attr'] = this.attr.toJson();
}
if (this.fields != null) {
data['fields'] = this.fields.map((v) => v.toJson()).toList();
}
return data;
}
}
class Attr {
String id;
Attr({this.id});
Attr.fromJson(Map<String, dynamic> json) {
id = json['id'];
}
Map<String, dynamic> toJson() {
final Map<String, dynamic> data = new Map<String, dynamic>();
data['id'] = this.id;
return data;
}
}
class Fields {
String node;
String name;
Attr attr;
Fields({this.node, this.name, this.attr});
Fields.fromJson(Map<String, dynamic> json) {
node = json['node'];
name = json['name'];
attr = json['attr'] != null ? new Attr.fromJson(json['attr']) : null;
}
Map<String, dynamic> toJson() {
final Map<String, dynamic> data = new Map<String, dynamic>();
data['node'] = this.node;
data['name'] = this.name;
if (this.attr != null) {
data['attr'] = this.attr.toJson();
}
return data;
}
}
class Attr {
String type;
String id;
String name;
String value;
String size;
String maxlength;
String class;
String title;
Attr({this.type, this.id, this.name, this.value, this.size, this.maxlength, this.class, this.title});
Attr.fromJson(Map<String, dynamic> json) {
type = json['type'];
id = json['id'];
name = json['name'];
value = json['value'];
size = json['size'];
maxlength = json['maxlength'];
class = json['class'];
title = json['title'];
}
Map<String, dynamic> toJson() {
final Map<String, dynamic> data = new Map<String, dynamic>();
data['type'] = this.type;
data['id'] = this.id;
data['name'] = this.name;
data['value'] = this.value;
data['size'] = this.size;
data['maxlength'] = this.maxlength;
data['class'] = this.class;
data['title'] = this.title;
return data;
}
}
class Tables {
Attr attr;
List<Rows> rows;
Tables({this.attr, this.rows});
Tables.fromJson(Map<String, dynamic> json) {
attr = json['attr'] != null ? new Attr.fromJson(json['attr']) : null;
if (json['rows'] != null) {
rows = new List<Rows>();
json['rows'].forEach((v) { rows.add(new Rows.fromJson(v)); });
}
}
Map<String, dynamic> toJson() {
final Map<String, dynamic> data = new Map<String, dynamic>();
if (this.attr != null) {
data['attr'] = this.attr.toJson();
}
if (this.rows != null) {
data['rows'] = this.rows.map((v) => v.toJson()).toList();
}
return data;
}
}
class Attr {
String id;
String width;
String cellspacing;
String style;
Attr({this.id, this.width, this.cellspacing, this.style});
Attr.fromJson(Map<String, dynamic> json) {
id = json['id'];
width = json['width'];
cellspacing = json['cellspacing'];
style = json['style'];
}
Map<String, dynamic> toJson() {
final Map<String, dynamic> data = new Map<String, dynamic>();
data['id'] = this.id;
data['width'] = this.width;
data['cellspacing'] = this.cellspacing;
data['style'] = this.style;
return data;
}
}
class Rows {
List<Null> attr;
List<Cols> cols;
Rows({this.attr, this.cols});
Rows.fromJson(Map<String, dynamic> json) {
if (json['attr'] != null) {
attr = new List<Null>();
json['attr'].forEach((v) { attr.add(new Null.fromJson(v)); });
}
if (json['cols'] != null) {
cols = new List<Cols>();
json['cols'].forEach((v) { cols.add(new Cols.fromJson(v)); });
}
}
Map<String, dynamic> toJson() {
final Map<String, dynamic> data = new Map<String, dynamic>();
if (this.attr != null) {
data['attr'] = this.attr.map((v) => v.toJson()).toList();
}
if (this.cols != null) {
data['cols'] = this.cols.map((v) => v.toJson()).toList();
}
return data;
}
}
class Cols {
List<Null> attr;
String nodeValue;
Cols({this.attr, this.nodeValue});
Cols.fromJson(Map<String, dynamic> json) {
if (json['attr'] != null) {
attr = new List<Null>();
json['attr'].forEach((v) { attr.add(new Null.fromJson(v)); });
}
nodeValue = json['nodeValue'];
}
Map<String, dynamic> toJson() {
final Map<String, dynamic> data = new Map<String, dynamic>();
if (this.attr != null) {
data['attr'] = this.attr.map((v) => v.toJson()).toList();
}
data['nodeValue'] = this.nodeValue;
return data;
}
}
class Imgs {
String src;
Attr attr;
Imgs({this.src, this.attr});
Imgs.fromJson(Map<String, dynamic> json) {
src = json['src'];
attr = json['attr'] != null ? new Attr.fromJson(json['attr']) : null;
}
Map<String, dynamic> toJson() {
final Map<String, dynamic> data = new Map<String, dynamic>();
data['src'] = this.src;
if (this.attr != null) {
data['attr'] = this.attr.toJson();
}
return data;
}
}
class Attr {
String src;
String style;
String title;
String border;
String align;
Attr({this.src, this.style, this.title, this.border, this.align});
Attr.fromJson(Map<String, dynamic> json) {
src = json['src'];
style = json['style'];
title = json['title'];
border = json['border'];
align = json['align'];
}
Map<String, dynamic> toJson() {
final Map<String, dynamic> data = new Map<String, dynamic>();
data['src'] = this.src;
data['style'] = this.style;
data['title'] = this.title;
data['border'] = this.border;
data['align'] = this.align;
return data;
}
}
class Links {
String text;
String href;
Attr attr;
Links({this.text, this.href, this.attr});
Links.fromJson(Map<String, dynamic> json) {
text = json['text'];
href = json['href'];
attr = json['attr'] != null ? new Attr.fromJson(json['attr']) : null;
}
Map<String, dynamic> toJson() {
final Map<String, dynamic> data = new Map<String, dynamic>();
data['text'] = this.text;
data['href'] = this.href;
if (this.attr != null) {
data['attr'] = this.attr.toJson();
}
return data;
}
}
class Attr {
String href;
String id;
String class;
String title;
String rel;
String dataOpenimage;
String dataClosedimage;
String target;
String style;
Attr({this.href, this.id, this.class, this.title, this.rel, this.dataOpenimage, this.dataClosedimage, this.target, this.style});
Attr.fromJson(Map<String, dynamic> json) {
href = json['href'];
id = json['id'];
class = json['class'];
title = json['title'];
rel = json['rel'];
dataOpenimage = json['data-openimage'];
dataClosedimage = json['data-closedimage'];
target = json['target'];
style = json['style'];
}
Map<String, dynamic> toJson() {
final Map<String, dynamic> data = new Map<String, dynamic>();
data['href'] = this.href;
data['id'] = this.id;
data['class'] = this.class;
data['title'] = this.title;
data['rel'] = this.rel;
data['data-openimage'] = this.dataOpenimage;
data['data-closedimage'] = this.dataClosedimage;
data['target'] = this.target;
data['style'] = this.style;
return data;
}
}