Compare commits

...

7 Commits

Author SHA1 Message Date
Augusto Gunsch f9f4c79ca2 Fix schema 2022-01-29 07:34:55 -03:00
Augusto Gunsch d7497218ff Remove development trash 2022-01-23 10:45:49 -03:00
Augusto Gunsch db3bea33e6 Improve multi language support 2022-01-23 10:37:38 -03:00
Augusto Gunsch 0ee20773bb Add frontend support 2022-01-23 00:24:23 -03:00
Augusto Gunsch 8d3ac4fc46
Add languages 2022-01-22 21:58:36 -03:00
Augusto Gunsch ea389f4b1c Improve state management 2022-01-12 08:04:19 -03:00
Augusto Gunsch 4e504997da Change way types are inserted 2022-01-10 13:24:55 -03:00
17 changed files with 409 additions and 158 deletions

View File

@ -1,6 +1,7 @@
use std::fs;
use std::io::ErrorKind;
use std::process::exit;
use std::fs::File;
use std::collections::HashSet;
use std::fmt;
use reqwest;
use rusqlite::{Connection, Transaction, ErrorCode};
@ -13,29 +14,83 @@ use serde_json;
use crate::language::Language;
use crate::entry::{WiktionaryEntries, WiktionaryEntry};
use crate::entry::Form;
use crate::{MAJOR, MINOR, PATCH};
use crate::{DB_DIR, CACHE_DIR, MAJOR, MINOR, PATCH};
use crate::util;
const DB_DIR: &str = "/usr/share/inflectived/";
const CACHE_DIR: &str = "/var/cache/";
pub enum DbError {
AccessDenied,
}
impl fmt::Display for DbError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
DbError::AccessDenied => write!(f, "Access denied"),
}
}
}
/// A database of Wiktionary entries
pub struct WordDb {
db_path: String
db_path: String,
pub installed_langs: Vec<Language>,
pub installable_langs: Vec<Language>
}
impl WordDb {
pub fn new(db_name: &str) -> Self {
let mut db_path = String::from(DB_DIR);
db_path.push_str(db_name);
let db_path = format!("{}/{}", DB_DIR, db_name);
Self { db_path }
let conn = Connection::open(&db_path).unwrap();
let mut installed_langs: Vec<Language> = Vec::new();
let statement = conn.prepare(
"SELECT code, name, major, minor, patch
FROM langs"
);
if let Ok(mut statement) = statement {
let mut rows = statement.query([]).unwrap();
while let Some(row) = rows.next().unwrap() {
installed_langs.push(Language::from_row(&row));
}
installed_langs.sort();
}
Self {
db_path,
installed_langs,
installable_langs: Language::list_langs(),
}
}
pub fn connect(&self) -> Connection {
Connection::open(&self.db_path).unwrap()
}
pub fn clean_tables(&mut self, lang: &Language) {
pub fn list_available(&self) -> String {
let mut list = String::new();
for lang in &self.installable_langs {
list.push_str(&format!(" - {} ({})\n", &lang.name, &lang.code));
}
list
}
pub fn get_lang(&self, code: &str) -> Option<Language> {
for lang in &self.installable_langs {
if lang.code == code {
return Some(lang.clone())
}
}
None
}
pub fn clean_tables(&mut self, lang: &Language) -> Result<(), DbError> {
let mut conn = self.connect();
let transaction = conn.transaction().unwrap();
@ -51,9 +106,7 @@ impl WordDb {
match e {
SqliteFailure(f, _) => match f.code {
ErrorCode::ReadOnly => {
eprintln!("Could not write to database: Permission denied");
eprintln!("Please run as root");
exit(1);
return Err(DbError::AccessDenied)
},
_ => panic!("{}", e)
},
@ -61,7 +114,18 @@ impl WordDb {
}
}
transaction.execute("DELETE FROM langs WHERE code = ?", [&lang.code]).unwrap();
if let Err(e) = transaction.execute(
"DELETE FROM langs WHERE code = ?", [&lang.code]) {
match e {
SqliteFailure(f, _) => match f.code {
ErrorCode::ReadOnly => {
return Err(DbError::AccessDenied)
},
_ => panic!("{}", e)
},
_ => panic!("{}", e)
}
};
transaction.execute(&format!("DROP TABLE IF EXISTS {0}_words", &lang.code), []).unwrap();
transaction.execute(&format!("DROP TABLE IF EXISTS {0}_types", &lang.code), []).unwrap();
@ -72,14 +136,6 @@ impl WordDb {
name TINYTEXT UNIQUE NOT NULL
)", &lang.code), []).unwrap();
for type_ in &lang.types {
transaction.execute(&format!("
INSERT INTO {0}_types ( name )
VALUES (
?
)", &lang.code), [type_]).unwrap();
}
transaction.execute(&format!("
CREATE TABLE {0}_words (
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
@ -91,16 +147,13 @@ impl WordDb {
)", &lang.code), []).unwrap();
transaction.execute(&format!("
CREATE INDEX word_index
CREATE INDEX {0}_word_index
ON {0}_words (word)
", &lang.code), []).unwrap();
transaction.execute("
INSERT INTO langs (code, name, major, minor, patch)
VALUES (?, ?, ?, ?, ?)
", params![&lang.code, &lang.name, MAJOR, MINOR, PATCH]).unwrap();
transaction.commit().unwrap();
Ok(())
}
pub fn insert_entry(&self, transaction: &Transaction, lang: &Language, entry: &WiktionaryEntry) {
@ -111,7 +164,7 @@ impl WordDb {
(SELECT id FROM {0}_types WHERE name = ?)
)", &lang.code),
params![entry.word,
entry.parsed_json.to_string(),
entry.unparsed_json,
entry.type_]
).unwrap();
}
@ -142,7 +195,7 @@ impl WordDb {
).unwrap();
for entry in entries.iter() {
if let Some(forms) = entry.parsed_json["forms"].as_array() {
if let Some(forms) = entry.parse_json()["forms"].as_array() {
let mut forms_vec: Vec<Form> = Vec::new();
for form in forms {
@ -169,7 +222,10 @@ impl WordDb {
let mut senses: Vec<Value> = Vec::new();
for form in forms {
let mut tags = form.tags.clone();
let mut tags = match &form.tags {
Some(tags) => tags.clone(),
None => Vec::new()
};
tags.push(String::from("form-of"));
tags.push(String::from("auto-generated"));
@ -180,7 +236,10 @@ impl WordDb {
}
],
"glosses": [
form.tags.join(" ")
match &form.tags {
Some(tags) => tags.join(" "),
None => String::from("")
}
],
"tags": tags
}));
@ -194,7 +253,7 @@ impl WordDb {
let new_entry = WiktionaryEntry::new(forms[0].form.clone(),
entry.type_.clone(),
entry_json);
entry_json.to_string());
self.insert_entry(&transaction, lang, &new_entry);
}
@ -206,57 +265,84 @@ impl WordDb {
transaction.commit().unwrap();
}
fn try_create_dir(&self, dir: &str) {
match fs::create_dir(dir) {
Err(e) => match e.kind() {
ErrorKind::AlreadyExists => {},
_ => panic!("{}", e)
},
_ => {}
}
fn insert_types(&mut self, lang: &Language, entries: &WiktionaryEntries) {
let mut conn = self.connect();
let transaction = conn.transaction().unwrap();
let mut types = HashSet::new();
for entry in entries.iter() {
types.insert(&entry.type_);
}
pub async fn upgrade_lang(&mut self, lang: &Language) {
self.try_create_dir(DB_DIR);
for type_ in types {
transaction.execute(&format!("
INSERT INTO {0}_types ( name )
VALUES (?)", &lang.code), [type_]).unwrap();
}
transaction.commit().unwrap();
}
fn insert_version(&mut self, lang: &Language) {
let mut conn = self.connect();
let transaction = conn.transaction().unwrap();
transaction.execute("
INSERT INTO langs (code, name, major, minor, patch)
VALUES (?, ?, ?, ?, ?)
", params![&lang.code, &lang.name, MAJOR, MINOR, PATCH]).unwrap();
transaction.commit().unwrap();
}
pub async fn upgrade_lang(&mut self, lang: &Language) -> Result<(), DbError> {
util::try_create_dir(DB_DIR);
println!("Trying to read cached data...");
let mut cache_file = String::from(CACHE_DIR);
cache_file.push_str("Polish.json");
let cache_file = format!("{}/{}.json", CACHE_DIR, &lang.name);
let cached_data = fs::read_to_string(&cache_file);
let mut cached_data = File::open(&cache_file);
let mut request = None;
if let Err(_) = cached_data {
request = Some(reqwest::get("https://kaikki.org/dictionary/Polish/kaikki.org-dictionary-Polish.json"));
let url = format!("https://kaikki.org/dictionary/{0}/kaikki.org-dictionary-{0}.json",
&lang.name);
request = Some(reqwest::get(url));
}
println!("Cleaning tables...");
self.clean_tables(lang);
self.clean_tables(lang)?;
let data;
if let Some(request) = request {
// Actually, the request was sent before
println!("Requesting data...");
data = request.await.unwrap().text().await.unwrap();
if cfg!(unix) {
let data = request.await.unwrap().text().await.unwrap();
println!("Caching data...");
self.try_create_dir(CACHE_DIR);
util::try_create_dir(CACHE_DIR);
fs::write(&cache_file, &data).unwrap();
}
}
else {
data = cached_data.unwrap();
cached_data = File::open(&cache_file);
}
println!("Parsing data...");
let entries = WiktionaryEntries::parse_data(data);
let entries = WiktionaryEntries::parse_data(cached_data.unwrap());
println!("Inserting data...");
println!("Inserting types...");
self.insert_types(lang, &entries);
println!("Inserting entries...");
self.insert_entries(lang, &entries);
println!("Generating \"form-of\" entries...");
self.generate_entries(lang, &entries);
println!("Inserting version...");
self.insert_version(lang);
println!("Done");
Ok(())
}
}

View File

@ -1,3 +1,5 @@
use std::fs::File;
use std::io::{BufRead, BufReader};
use std::cmp;
use std::slice::Iter;
use serde_json::Value;
@ -7,7 +9,7 @@ use serde::Deserialize;
pub struct WiktionaryEntry {
pub word: String,
pub type_: String,
pub parsed_json: Value,
pub unparsed_json: String
}
impl cmp::PartialEq for WiktionaryEntry {
@ -32,6 +34,8 @@ impl cmp::Ord for WiktionaryEntry {
impl WiktionaryEntry {
pub fn parse(unparsed_json: &str) -> Self {
// We could keep this in memory, but for bigger language databases
// it's going to crash the program
let json: Value = serde_json::from_str(unparsed_json).unwrap();
let word = String::from(json["word"].as_str().unwrap());
@ -40,27 +44,33 @@ impl WiktionaryEntry {
Self {
word,
type_,
parsed_json: json
unparsed_json: String::from(unparsed_json)
}
}
pub fn new(word: String, type_: String, parsed_json: Value) -> Self {
pub fn new(word: String, type_: String, unparsed_json: String) -> Self {
Self {
word,
type_,
parsed_json
unparsed_json
}
}
pub fn parse_json(&self) -> Value {
serde_json::from_str(&self.unparsed_json).unwrap()
}
}
pub struct WiktionaryEntries(Vec<WiktionaryEntry>);
impl WiktionaryEntries {
pub fn parse_data(data: String) -> Self {
pub fn parse_data(data: File) -> Self {
let reader = BufReader::new(data);
let mut entries: Vec<WiktionaryEntry> = Vec::new();
for line in data.lines() {
entries.push(WiktionaryEntry::parse(line));
for line in reader.lines() {
entries.push(WiktionaryEntry::parse(&line.unwrap()));
}
Self(entries)
@ -74,7 +84,7 @@ impl WiktionaryEntries {
#[derive(Debug, Deserialize)]
pub struct Form {
pub form: String,
pub tags: Vec<String>,
pub tags: Option<Vec<String>>,
pub source: Option<String>,
}

View File

@ -1,16 +1,69 @@
#[derive(Debug)]
use std::cmp::{PartialEq, PartialOrd, Ordering};
use rusqlite::Row;
use serde::Serialize;
use crate::version::Version;
#[derive(Serialize, Debug, Clone)]
pub struct Language {
pub code: String,
pub name: String,
pub types: Vec<String>
pub code: String, // ISO 639-2
pub name: String, // English name
pub version: Option<Version>
}
impl Language {
pub fn new(code: &str, name: &str, types: Vec<String>) -> Self {
pub fn new(code: &str, name: &str) -> Self {
Self {
code: String::from(code),
name: String::from(name),
types
version: None
}
}
pub fn from_row(row: &Row) -> Self {
Self {
code: row.get(0).unwrap(),
name: row.get(1).unwrap(),
version: Some(Version(row.get(2).unwrap(),
row.get(3).unwrap(),
row.get(4).unwrap()))
}
}
pub fn list_langs() -> Vec<Self> {
// Keep this list sorted by name
let langs = vec![
Self::new("eng", "English"),
Self::new("fre", "French"),
Self::new("ger", "German"),
Self::new("ita", "Italian"),
Self::new("pol", "Polish"),
Self::new("por", "Portuguese"),
Self::new("rus", "Russian"),
Self::new("spa", "Spanish"),
];
langs
}
}
impl PartialEq for Language {
fn eq(&self, other: &Self) -> bool {
self.name == other.name
}
}
impl Eq for Language {}
impl Ord for Language {
fn cmp(&self, other: &Self) -> Ordering {
self.name.cmp(&other.name)
}
}
impl PartialOrd for Language {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
Some(self.name.cmp(&other.name))
}
}

View File

@ -1,18 +1,26 @@
#![feature(slice_group_by)]
#![feature(slice_group_by, path_try_exists)]
use std::process::exit;
use std::fs;
//mod database;
use rocket::routes;
use rocket::fs::FileServer;
use clap::{App, AppSettings, Arg, SubCommand};
//use database::WordDb;
mod database;
mod language;
mod entry;
mod views;
mod version;
mod util;
use database::WordDb;
use language::Language;
use database::{WordDb, DbError};
const DB_DIR: &str = "/usr/share/inflectived";
const CACHE_DIR: &str = "/var/cache/inflectived";
const FRONTEND_DIR: &str = "/opt/inflectived";
const MAJOR: i32 = 0;
const MINOR: i32 = 1;
@ -56,46 +64,42 @@ async fn main() {
let mut db = WordDb::new("inflectived.db");
let lang = Language::new("pl",
"Polish",
vec![String::from("adj"),
String::from("noun"),
String::from("verb"),
String::from("character"),
String::from("suffix"),
String::from("prefix"),
String::from("conj"),
String::from("adv"),
String::from("infix"),
String::from("name"),
String::from("phrase"),
String::from("prep_phrase"),
String::from("intj"),
String::from("det"),
String::from("prep"),
String::from("proverb"),
String::from("abbrev"),
String::from("num"),
String::from("pron"),
String::from("punct"),
String::from("interfix"),
String::from("particle")]);
match matches.subcommand() {
("upgrade", _) => { db.upgrade_lang(&lang).await; },
("run", _) => {
("upgrade", matches) => {
let lang = db.get_lang(matches.unwrap().value_of("LANG").unwrap());
if let None = lang {
eprintln!("The requested language is not available.");
eprintln!("Available languages:");
eprint!("{}", db.list_available());
exit(1);
}
if let Err(e) = db.upgrade_lang(&lang.unwrap()).await {
match e {
DbError::AccessDenied => {
eprintln!("Permission denied. Please run as root.");
exit(1);
}
}
}
},
("run", _matches) => {
let figment = rocket::Config::figment()
.merge(("address", "0.0.0.0"));
rocket::custom(figment)
let mut app = rocket::custom(figment)
.manage(db)
.mount("/static", FileServer::from("static/"))
.mount("/", routes![views::get_entries,
views::get_entries_like,
views::get_langs,
views::frontend])
.launch()
.await.unwrap();
views::frontend]);
if let Ok(_) = fs::try_exists(FRONTEND_DIR) {
app = app.mount("/static", FileServer::from(FRONTEND_DIR));
}
app.launch().await.unwrap();
},
_ => {}
}

12
src/util.rs Normal file
View File

@ -0,0 +1,12 @@
use std::fs;
use std::io::ErrorKind;
pub fn try_create_dir(dir: &str) {
match fs::create_dir(dir) {
Err(e) => match e.kind() {
ErrorKind::AlreadyExists => {},
_ => panic!("{}", e)
},
_ => {}
}
}

36
src/version.rs Normal file
View File

@ -0,0 +1,36 @@
use std::cmp::{PartialEq, PartialOrd, Ordering};
use serde::Serialize;
#[derive(Serialize, Debug, Clone, PartialEq)]
pub struct Version(pub u32, pub u32, pub u32);
impl PartialOrd for Version {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
if self.0 > other.0 {
return Some(Ordering::Greater);
}
if self.0 < other.0 {
return Some(Ordering::Less);
}
if self.1 > other.1 {
return Some(Ordering::Greater);
}
if self.1 < other.1 {
return Some(Ordering::Less);
}
if self.2 > other.2 {
return Some(Ordering::Greater);
}
if self.2 < other.2 {
return Some(Ordering::Less);
}
Some(Ordering::Equal)
}
}

View File

@ -8,12 +8,14 @@ use rocket::serde::json::Json;
use rusqlite::params;
use crate::database::WordDb;
use crate::language::Language;
use crate::FRONTEND_DIR;
#[get("/")]
pub fn frontend() -> Option<content::Html<String>> {
match fs::read_to_string("static/index.html") {
Ok(file) => Some(content::Html(file)),
Err(_) => None
pub fn frontend() -> content::Html<String> {
match fs::read_to_string(&format!("{}/{}", FRONTEND_DIR, "index.html")) {
Ok(file) => content::Html(file),
Err(_) => content::Html(String::from("<h1>No web frontend installed.</h1>"))
}
}
@ -72,18 +74,16 @@ pub fn get_entries_like(db: &State<WordDb>, lang: &str, like: &str, limit: usize
}
#[get("/langs?<installed>")]
pub fn get_langs(db: &State<WordDb>, installed: bool) -> Json<Vec<String>> {
let conn = db.connect();
let mut langs: Vec<String> = Vec::new();
pub fn get_langs(db: &State<WordDb>, installed: bool) -> Json<Vec<Language>> {
let mut langs: Vec<Language> = Vec::new();
if installed {
let mut statement = conn.prepare("SELECT name FROM langs").unwrap();
let mut rows = statement.query([]).unwrap();
while let Some(row) = rows.next().unwrap() {
langs.push(row.get(0).unwrap());
for lang in &db.installed_langs {
langs.push(lang.clone())
}
} else {
for lang in &db.installable_langs {
langs.push(lang.clone())
}
}

View File

@ -2,7 +2,7 @@
<html>
<head>
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
<meta charset="utf-8"/>
<link rel="icon" href="/static/favicon.png">
<link rel="stylesheet" href="static/jquery-ui-1.13.0/jquery-ui.min.css">
<link rel="stylesheet" href="static/jquery-ui-1.13.0/jquery-ui.theme.min.css">
@ -19,9 +19,9 @@
<div class="ui-widget" id="search-widget">
<input id="search-bar" placeholder="Search..." type="text" autocorrect="off" autocapitalize="none" autofocus>
</div>
<button type="submit" class="ui-button ui-widget" id="search">
<span class="ui-icon ui-icon-search"></span>
</button>
<select id="langs">
</select>
<!--<button id="menu"></button>-->
</form>
<div class="container" id="ajax-content">
</div>

View File

@ -1,8 +1,42 @@
$(document).ready(() => {
let polishSchemas = null;
let selectedLang = null;
let schema = null;
let langs = null;
$.ajax({
url: '/static/schemas/polish.json',
url: `/langs?installed`,
success: data => {
langs = data;
const selectedLangCode = localStorage.selectedLangCode;
let options = '';
langs.forEach(lang => {
if(selectedLangCode && lang.code == selectedLangCode) {
options += `<option value="${lang.code}" selected>${lang.name}</option>`;
} else {
options += `<option value="${lang.code}">${lang.name}</option>`;
}
});
$('#langs').html(options);
setLang($('#langs').val());
}
});
$('#langs').on('change', e => {
setLang(e.target.value);
});
function setLang(code) {
const lang = langs.find(lang => lang.code == code);
localStorage.selectedLangCode = code;
selectedLang = lang;
$.ajax({
url: `/static/schemas/${lang.name}.json`,
success: data => {
polishSchemas = data
if(window.location.hash) {
@ -10,18 +44,21 @@ $(document).ready(() => {
}
}
});
}
const searchBar = $('#search-bar');
const searchForm = $('#search-form');
const ajaxContent = $('#ajax-content');
window.onhashchange = () => {
getWord();
};
const searchBar = $('#search-bar');
searchBar.autocomplete({
appendTo: '#search-form',
source: (request, response) => {
$.ajax({
url: '/langs/pl/words?like=' + request.term + '&limit=20&offset=0',
url: `/langs/${selectedLang.code}/words?like=${request.term}&limit=20&offset=0`,
success: data => response(data)
})
},
@ -32,22 +69,25 @@ $(document).ready(() => {
setTimeout(() => e.currentTarget.select(), 100);
});
$('#search-form').on('submit', e => {
searchForm.on('submit', e => {
e.preventDefault();
let word = e.target[0].value
const word = e.target[0].value
window.location.hash = `#${word}`;
});
function getWord() {
let word = window.location.hash.replace('#', '');
const word = window.location.hash.replace('#', '');
if (word) {
document.title = `Inflective - ${decodeURIComponent(word)}`;
$.ajax({
url: '/langs/pl/words/' + word,
url: `/langs/${selectedLang.code}/words/${word}`,
success: (data) => {
$('#ajax-content').html(generateHtml(word, data))
ajaxContent.html(generateHtml(word, data));
},
error: err => console.error(err)
@ -59,6 +99,9 @@ $(document).ready(() => {
// Sometimes autocomplete opens after close was called
// A better fix should be made
setTimeout(() => searchBar.autocomplete('close'), 1000);
} else {
ajaxContent.html('');
}
}
function getCells(forms, tags) {

View File

@ -0,0 +1 @@
[]

View File

@ -0,0 +1 @@
[]

View File

@ -0,0 +1 @@
[]

View File

@ -0,0 +1 @@
[]

View File

@ -385,12 +385,12 @@
{
"colspan": 1,
"rowspan": 1,
"tags": ["masculine", "inanimate", "singular", "accusative"]
"tags": ["masculine", "animate", "singular", "accusative"]
},
{
"colspan": 1,
"rowspan": 1,
"tags": ["masculine", "animate", "singular", "accusative"]
"tags": ["masculine", "inanimate", "singular", "accusative"]
},
{
"colspan": 1,

View File

@ -0,0 +1 @@
[]

View File

@ -0,0 +1 @@
[] Italian.json

View File

@ -0,0 +1 @@
[]