Skip to content
Snippets Groups Projects
Commit fca96fc6 authored by Tom Seimandi's avatar Tom Seimandi
Browse files

Change s3 paths

parent a64bff3f
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
.gitignore
+ 1
− 2
View file @ fca96fc6
......@@ -2,6 +2,5 @@
.Rhistory
.RData
.Ruserdata
donnees/era5/*
data/era5/*
get-pip.py
Sujet2-Etape1.R
+ 8
− 8
View file @ fca96fc6
......@@ -28,7 +28,7 @@ renv::restore()
# données -----------------------------------------------------------------
aws.s3::get_bucket("projet-funathon", region = "", prefix = "2023/sujet2/ign/rpg")
aws.s3::get_bucket("projet-funathon", region = "", prefix = "2023/diffusion/sujet2/ign/rpg")
# 1 - Créer une table avec un point en récupérant les coordonnées GPS --------
......@@ -112,7 +112,7 @@ plot(st_geometry(parc_prox))
# 5 - Lecture et appariement des cultures agrégées ---------------------------------------
cult_agreg<-s3read_using(FUN = read_csv,
object = "2023/sujet2/ign/rpg/n-cultures-2021.csv",
object = "2023/sujet2/diffusion/ign/rpg/n-cultures-2021.csv",
col_types = cols(.default = col_character()),
bucket = "projet-funathon",
opts = list("region" = "")) %>% select(-nom_sous_chapitre,-categorie_surf_agricole)
......@@ -124,7 +124,7 @@ parc_prox <- parc_prox %>% left_join(cult_agreg,by=c("code_cultu"="code_culture"
# 5 - lecture des libelles des groupes de culture --------------------------------------
lib_group_cult<-s3read_using(FUN = read_csv2,
object = "2023/sujet2/ign/rpg/REF_CULTURES_GROUPES_CULTURES_2020.csv",
object = "2023/sujet2/diffusion/ign/rpg/REF_CULTURES_GROUPES_CULTURES_2020.csv",
col_types = cols(.default = col_character()),
bucket = "projet-funathon",
opts = list("region" = "")) %>%
......@@ -147,12 +147,12 @@ stat_group_cult_fm<-stat_sql_group_cult_fm %>%
s3saveRDS(stat_group_cult_fm,
bucket = "projet-funathon",
object = "/2023/sujet2/resultats/stat_group_cult_fm.rds",
object = "/2023/sujet2/diffusion/resultats/stat_group_cult_fm.rds",
opts = list("region" = ""))
s3write_using(stat_group_cult_fm,
FUN = write_csv,
object = "/2023/sujet2/resultats/stat_group_cult_fm.csv",
object = "/2023/sujet2/diffusion/resultats/stat_group_cult_fm.csv",
bucket = "projet-funathon",
opts = list("region" = ""))
......@@ -207,7 +207,7 @@ stat_group_cult_dep<-stat_sql_group_cult_dep %>%
s3write_using(stat_group_cult_dep,
FUN = write_csv,
object = "/2023/sujet2/resultats/stat_group_cult_dep.csv",
object = "/2023/sujet2/diffusion/resultats/stat_group_cult_dep.csv",
bucket = "projet-funathon",
opts = list("region" = ""))
......@@ -271,13 +271,13 @@ stat_group_cult_by_dep<-stats_group_cult_by_dep %>%
s3write_using(stat_group_cult_by_dep,
FUN = write_csv,
object = "/2023/sujet2/resultats/stat_group_cult_by_dep.csv",
object = "/2023/sujet2/diffusion/resultats/stat_group_cult_by_dep.csv",
bucket = "projet-funathon",
opts = list("region" = ""))
s3write_using(
stat_group_cult_by_dep,
readr::write_rds,
object = "2023/sujet2/resultats/stat_group_cult_by_dep.rds",
object = "2023/sujet2/diffusion/resultats/stat_group_cult_by_dep.rds",
bucket = "projet-funathon",
opts = list("region" = ""))
......
Sujet2-Etape1.qmd
+ 5
− 5
View file @ fca96fc6
......@@ -27,7 +27,7 @@ if (!"kableExtra" %in% installed.packages()) { install.packages("kableExtra") }
library(kableExtra)
# données -----------------------------------------------------------------
aws.s3::get_bucket("projet-funathon", region = "", prefix = "2023/sujet2/ign/rpg")
aws.s3::get_bucket("projet-funathon", region = "", prefix = "2023/sujet2/diffusion/ign/rpg")
```
......@@ -149,7 +149,7 @@ plot(st_geometry(point),add = T,col = "red")
#| label: lecture libellés groupes cultures
lib_group_cult<-s3read_using(FUN = read_csv2,
object = "2023/sujet2/ign/rpg/REF_CULTURES_GROUPES_CULTURES_2020.csv",
object = "2023/sujet2/diffusion/ign/rpg/REF_CULTURES_GROUPES_CULTURES_2020.csv",
col_types = cols(.default = col_character()),
bucket = "projet-funathon",
opts = list("region" = "")) %>%
......@@ -208,7 +208,7 @@ rm(t1)
# com <- s3read_using(
# FUN = sf::read_sf,
# layer = "COMMUNE_2021",
# object = "2023/sujet2/ign/COMMUNE_2021.gpkg",
# object = "2023/sujet2/diffusion/ign/COMMUNE_2021.gpkg",
# bucket = "projet-funathon",
# opts = list("region" = "")) %>% clean_names()
......@@ -234,7 +234,7 @@ stat_pt <- parc_prox %>% st_drop_geometry() %>%
# récup des % surfaces départementales
stat_dep_pt<-s3read_using(
FUN=readr::read_rds,
object = "2023/sujet2/resultats/stat_group_cult_by_dep.rds",
object = "2023/sujet2/diffusion/resultats/stat_group_cult_by_dep.rds",
bucket = "projet-funathon",
opts = list("region" = "")) %>%
filter(insee_dep %in% dep_pt) %>%
......@@ -244,7 +244,7 @@ stat_dep_pt<-s3read_using(
# récup des % surfaces nationales
stat_fm<-s3read_using(
FUN=readr::read_csv,
object = "2023/sujet2/resultats/stat_group_cult_fm.csv",
object = "2023/sujet2/diffusion/resultats/stat_group_cult_fm.csv",
col_types = cols(code_group = col_character()),
bucket = "projet-funathon",
opts = list("region" = "")) %>%
......
app2.R
+ 6
− 6
View file @ fca96fc6
......@@ -47,7 +47,7 @@ drias <- s3read_using(
"NORRRA", "NORSTM6", "NORSTM0", "NORSDA", "NORDATEVEG",
"NORDATEDG", "NORDATEPG", "ARRA", "ASTM6", "ASTM0",
"ASDA", "ADATEVEG", "ADATEDG", "ADATEPG"),
object = "2023/sujet2/drias/indicesALADIN63_CNRM-CM5_23050511192547042.KEYuUdx3UA39Av7f1U7u7O.txt",
object = "2023/sujet2/diffusion/drias/indicesALADIN63_CNRM-CM5_23050511192547042.KEYuUdx3UA39Av7f1U7u7O.txt",
bucket = "projet-funathon",
opts = list("region" = "")) %>%
janitor::clean_names() %>%
......@@ -88,7 +88,7 @@ precip_df <- precip_df %>%
culture_mapping <- s3read_using(
FUN = read.csv,
sep = ";",
object = "2023/sujet2/ign/rpg/CULTURE.csv",
object = "2023/sujet2/diffusion/ign/rpg/CULTURE.csv",
bucket = "projet-funathon",
opts = list("region" = "")
)
......@@ -102,11 +102,11 @@ precip_df %>%
arrange(ecart_cumul_moyen)
# Illustration, on passe par du raster pour plot ?
aws.s3::get_bucket("projet-funathon", region = "", prefix = "2023/sujet2/resultats")
aws.s3::get_bucket("projet-funathon", region = "", prefix = "2023/sujet2/diffusion/resultats")
drias_raster <- s3read_using(
function(f) readAll(brick(f)),
object = "2023/sujet2/resultats/drias.tif",
object = "2023/sujet2/diffusion/resultats/drias.tif",
bucket = "projet-funathon",
opts = list("region" = ""))
drias_df <- as.data.frame(drias_raster, xy = TRUE) %>% tidyr::drop_na()
......@@ -137,7 +137,7 @@ bandnr(drias_raster)
# Bande ARRA
drias_raster_arra <- s3read_using(
function(f) readAll(raster(f, band = 8)),
object = "2023/sujet2/resultats/drias.tif",
object = "2023/sujet2/diffusion/resultats/drias.tif",
bucket = "projet-funathon",
opts = list("region" = ""))
as.data.frame(drias_raster_arra, xy = TRUE)
......@@ -161,7 +161,7 @@ raster::plot(x = drias_raster_arra,
# Bande ASDA
drias_raster_asda <- s3read_using(
function(f) readAll(raster(f, band = 11)),
object = "2023/sujet2/resultats/drias.tif",
object = "2023/sujet2/diffusion/resultats/drias.tif",
bucket = "projet-funathon",
opts = list("region" = ""))
......
db.R
+ 4
− 4
View file @ fca96fc6
......@@ -18,12 +18,12 @@ cnx <- dbConnect(Postgres(),
error = function(e) ''),
sep = " - "))
aws.s3::get_bucket("projet-funathon", region = "", prefix = "2023/sujet2")
aws.s3::get_bucket("projet-funathon", region = "", prefix = "2023/sujet2/diffusion")
# couches dispo dans le gpkg
s3read_using(
FUN = sf::st_layers,
object = "2023/sujet2/ign/adminexpress_cog_simpl_000_2023.gpkg",
object = "2023/sujet2/diffusion/ign/adminexpress_cog_simpl_000_2023.gpkg",
bucket = "projet-funathon",
opts = list("region" = "")
)
......@@ -32,7 +32,7 @@ s3read_using(
reg <- s3read_using(
FUN = sf::read_sf,
layer = "region",
object = "2023/sujet2/ign/adminexpress_cog_simpl_000_2023.gpkg",
object = "2023/sujet2/diffusion/ign/adminexpress_cog_simpl_000_2023.gpkg",
bucket = "projet-funathon",
opts = list("region" = ""))
......@@ -50,7 +50,7 @@ EPSG:WGS84$$")
com <- s3read_using(
FUN = sf::read_sf,
layer = "commune",
object = "2023/sujet2/ign/adminexpress_cog_simpl_000_2023.gpkg",
object = "2023/sujet2/diffusion/ign/adminexpress_cog_simpl_000_2023.gpkg",
bucket = "projet-funathon",
opts = list("region" = ""))
......
drias.R
+ 7
− 7
View file @ fca96fc6
......@@ -14,7 +14,7 @@ library(RPostgres)
# données -----------------------------------------------------------------
aws.s3::get_bucket("projet-funathon", region = "", prefix = "2023/sujet2")
aws.s3::get_bucket("projet-funathon", region = "", prefix = "2023/sujet2/diffusion")
# http://www.drias-climat.fr/commande
......@@ -29,7 +29,7 @@ drias <- s3read_using(
"NORRRA", "NORSTM6", "NORSTM0", "NORSDA", "NORDATEVEG",
"NORDATEDG", "NORDATEPG", "ARRA", "ASTM6", "ASTM0",
"ASDA", "ADATEVEG", "ADATEDG", "ADATEPG"),
object = "2023/sujet2/drias/indicesALADIN63_CNRM-CM5_23050511192547042.KEYuUdx3UA39Av7f1U7u7O.txt",
object = "2023/sujet2/diffusion/drias/indicesALADIN63_CNRM-CM5_23050511192547042.KEYuUdx3UA39Av7f1U7u7O.txt",
bucket = "projet-funathon",
opts = list("region" = "")) %>%
clean_names() %>%
......@@ -48,7 +48,7 @@ grille <- s3read_using(
col_names = c("id", "x_n", "y_n", "x_l2e", "y_l2e",
"x_l93", "y_l93", "lon", "lat",
"alti", "unused"),
object = "2023/sujet2/drias/grilleSafran_complete_drias2021.xls",
object = "2023/sujet2/diffusion/drias/grilleSafran_complete_drias2021.xls",
bucket = "projet-funathon",
opts = list("region" = "")) %>%
dplyr::select(-unused)
......@@ -57,7 +57,7 @@ grille <- s3read_using(
fr <- s3read_using(
FUN = sf::read_sf,
layer = "region",
object = "2023/sujet2/ign/adminexpress_cog_simpl_000_2023.gpkg",
object = "2023/sujet2/diffusion/ign/adminexpress_cog_simpl_000_2023.gpkg",
bucket = "projet-funathon",
opts = list("region" = "")) %>%
filter(insee_reg > "06") %>%
......@@ -78,7 +78,7 @@ drias_sf <- drias %>%
drias_sf %>%
aws.s3::s3write_using(
sf::write_sf,
object = "2023/sujet2/resultats/drias.gpkg",
object = "2023/sujet2/diffusion/resultats/drias.gpkg",
bucket = "projet-funathon",
opts = list("region" = ""))
......@@ -97,14 +97,14 @@ drias_raster %>%
aws.s3::s3write_using(
raster::writeRaster,
overwrite = TRUE,
object = "2023/sujet2/resultats/drias.tif",
object = "2023/sujet2/diffusion/resultats/drias.tif",
bucket = "projet-funathon",
opts = list("region" = ""))
drias_raster %>%
aws.s3::s3write_using(
readr::write_rds,
object = "2023/sujet2/resultats/drias.rds",
object = "2023/sujet2/diffusion/resultats/drias.rds",
bucket = "projet-funathon",
opts = list("region" = ""))
......
era5.R
+ 3
− 3
View file @ fca96fc6
......@@ -91,17 +91,17 @@ periode %>%
# Mon compte > Connexion au stockage > Pour accéder au stockage > MC client
# $ export MC_HOST_s3=...
# puis par ex. :
# $ mc cp -r funathon2023_sujet2/donnees/era5/2022/ s3/projet-funathon/2023/sujet2/era5/2022
# $ mc cp -r funathon2023_sujet2/data/era5/2022/ s3/projet-funathon/2023/sujet2/diffusion/era5/2022
# ne marche pas (?) :
# periode %>%
# walk(~ system(glue("export MC_HOST_s3=\"{Sys.getenv('MC_HOST_s3')}\" && mc cp -r {rep_era5_full}{.x}/ s3/projet-funathon/2023/sujet2/era5/{.x}")))
# walk(~ system(glue("export MC_HOST_s3=\"{Sys.getenv('MC_HOST_s3')}\" && mc cp -r {rep_era5_full}{.x}/ s3/projet-funathon/2023/sujet2/diffusion/era5/{.x}")))
# ex. utilisation ---------------------------------------------------------
# copier les données depuis le stockage persistant S3
system(glue("mc cp -r s3/projet-funathon/2023/sujet2/era5/ {rep_era5_full}"))
system(glue("mc cp -r s3/projet-funathon/2023/sujet2/diffusion/era5/ {rep_era5_full}"))
# un exemple de localisations avec date début-fin
points <- tribble(~ville, ~lon, ~lat,
......
init.sh
+ 5
− 0
View file @ fca96fc6
......@@ -6,3 +6,8 @@ chown -R onyxia:users $PROJECT_DIR/
cd $PROJECT_DIR
git config --global credential.helper store
# s3 data
mc cp s3/projet-funathon/2023/sujet2/diffusion/era5.zip ~/work/funathon2023_sujet2/data/era5.zip
unzip ~/work/funathon2023_sujet2/data/era5.zip -d ~/work/funathon2023_sujet2/data/
rm ~/work/funathon2023_sujet2/data/era5.zip
rpg.R
+ 6
− 6
View file @ fca96fc6
......@@ -4,7 +4,7 @@ library(sf)
library(aws.s3)
library(RPostgres)
aws.s3::get_bucket("projet-funathon", region = "", prefix = "2023/sujet2/ign/rpg")
aws.s3::get_bucket("projet-funathon", region = "", prefix = "2023/sujet2/diffusion/ign/rpg")
# Ilots
# Un îlot de culture correspond à un groupe de parcelles contiguës,
......@@ -13,7 +13,7 @@ aws.s3::get_bucket("projet-funathon", region = "", prefix = "2023/sujet2/ign/rpg
# Couches dispo dans le .gpkg
s3read_using(
FUN = sf::st_layers,
object = "2023/sujet2/ign/rpg/ILOTS_ANONYMES.gpkg",
object = "2023/sujet2/diffusion/ign/rpg/ILOTS_ANONYMES.gpkg",
bucket = "projet-funathon",
opts = list("region" = "")
)
......@@ -22,7 +22,7 @@ s3read_using(
ilots <- s3read_using(
FUN = sf::read_sf,
query = 'SELECT * FROM ilots_anonymes LIMIT 10',
object = "2023/sujet2/ign/rpg/ILOTS_ANONYMES.gpkg",
object = "2023/sujet2/diffusion/ign/rpg/ILOTS_ANONYMES.gpkg",
bucket = "projet-funathon",
opts = list("region" = "")
)
......@@ -33,7 +33,7 @@ ilots <- s3read_using(
# Couches dispo dans le .gpkg
s3read_using(
FUN = sf::st_layers,
object = "2023/sujet2/ign/rpg/PARCELLES_GRAPHIQUES.gpkg",
object = "2023/sujet2/diffusion/ign/rpg/PARCELLES_GRAPHIQUES.gpkg",
bucket = "projet-funathon",
opts = list("region" = "")
)
......@@ -42,7 +42,7 @@ s3read_using(
parcelles <- s3read_using(
FUN = sf::read_sf,
query = 'SELECT * FROM parcelles_graphiques LIMIT 10',
object = "2023/sujet2/ign/rpg/PARCELLES_GRAPHIQUES.gpkg",
object = "2023/sujet2/diffusion/ign/rpg/PARCELLES_GRAPHIQUES.gpkg",
bucket = "projet-funathon",
opts = list("region" = "")
)
......@@ -89,7 +89,7 @@ for (offset in offsets) {
parcelles_part <- s3read_using(
FUN = sf::read_sf,
query = query,
object = "2023/sujet2/ign/rpg/PARCELLES_GRAPHIQUES.gpkg",
object = "2023/sujet2/diffusion/ign/rpg/PARCELLES_GRAPHIQUES.gpkg",
bucket = "projet-funathon",
opts = list("region" = "")
)
......
website/app0.qmd
+ 1
− 1
View file @ fca96fc6
......@@ -85,7 +85,7 @@ drias <- s3read_using(
"NORRRA", "NORSTM6", "NORSTM0", "NORSDA", "NORDATEVEG",
"NORDATEDG", "NORDATEPG", "ARRA", "ASTM6", "ASTM0",
"ASDA", "ADATEVEG", "ADATEDG", "ADATEPG"),
object = "2023/sujet2/drias/indicesALADIN63_CNRM-CM5_23050511192547042.KEYuUdx3UA39Av7f1U7u7O.txt",
object = "2023/sujet2/diffusion/drias/indicesALADIN63_CNRM-CM5_23050511192547042.KEYuUdx3UA39Av7f1U7u7O.txt",
bucket = "projet-funathon",
opts = list("region" = "")) %>%
clean_names() %>%
......
website/app1.qmd
+ 5
− 5
View file @ fca96fc6
......@@ -27,7 +27,7 @@ if (!"kableExtra" %in% installed.packages()) { install.packages("kableExtra") }
library(kableExtra)
# données -----------------------------------------------------------------
aws.s3::get_bucket("projet-funathon", region = "", prefix = "2023/sujet2/ign/rpg")
aws.s3::get_bucket("projet-funathon", region = "", prefix = "2023/sujet2/diffusion/ign/rpg")
```
......@@ -149,7 +149,7 @@ plot(st_geometry(point),add = T,col = "red")
#| label: lecture libellés groupes cultures
lib_group_cult<-s3read_using(FUN = read_csv2,
object = "2023/sujet2/ign/rpg/REF_CULTURES_GROUPES_CULTURES_2020.csv",
object = "2023/sujet2/diffusion/ign/rpg/REF_CULTURES_GROUPES_CULTURES_2020.csv",
col_types = cols(.default = col_character()),
bucket = "projet-funathon",
opts = list("region" = "")) %>%
......@@ -208,7 +208,7 @@ rm(t1)
# com <- s3read_using(
# FUN = sf::read_sf,
# layer = "COMMUNE_2021",
# object = "2023/sujet2/ign/COMMUNE_2021.gpkg",
# object = "2023/sujet2/diffusion/ign/COMMUNE_2021.gpkg",
# bucket = "projet-funathon",
# opts = list("region" = "")) %>% clean_names()
......@@ -234,7 +234,7 @@ stat_pt <- parc_prox %>% st_drop_geometry() %>%
# récup des % surfaces départementales
stat_dep_pt<-s3read_using(
FUN=readr::read_rds,
object = "2023/sujet2/resultats/stat_group_cult_by_dep.rds",
object = "2023/sujet2/diffusion/resultats/stat_group_cult_by_dep.rds",
bucket = "projet-funathon",
opts = list("region" = "")) %>%
filter(insee_dep %in% dep_pt) %>%
......@@ -244,7 +244,7 @@ stat_dep_pt<-s3read_using(
# récup des % surfaces nationales
stat_fm<-s3read_using(
FUN=readr::read_csv,
object = "2023/sujet2/resultats/stat_group_cult_fm.csv",
object = "2023/sujet2/diffusion/resultats/stat_group_cult_fm.csv",
col_types = cols(code_group = col_character()),
bucket = "projet-funathon",
opts = list("region" = "")) %>%
......
website/app2.qmd
+ 3
− 3
View file @ fca96fc6
......@@ -53,7 +53,7 @@ On souhaite tout d'abord visualiser les données DRIAS. On peut passer par les d
drias_raster <- s3read_using(
function(f) readAll(brick(f)),
object = "2023/sujet2/resultats/drias.tif",
object = "2023/sujet2/diffusion/resultats/drias.tif",
bucket = "projet-funathon",
opts = list("region" = ""))
......@@ -98,7 +98,7 @@ On visualise la variable ARRA, qui correspond à l'écart de cumul de précipita
# Bande ARRA
drias_raster_arra <- s3read_using(
function(f) readAll(raster(f, band = 8)),
object = "2023/sujet2/resultats/drias.tif",
object = "2023/sujet2/diffusion/resultats/drias.tif",
bucket = "projet-funathon",
opts = list("region" = ""))
......@@ -168,7 +168,7 @@ On peut maintenant calculer un écart moyen de cumul de précipitation (d'avril
culture_mapping <- s3read_using(
FUN = read.csv,
sep = ";",
object = "2023/sujet2/ign/rpg/CULTURE.csv",
object = "2023/sujet2/diffusion/ign/rpg/CULTURE.csv",
bucket = "projet-funathon",
opts = list("region" = "")
)
......
website/physio_era5.qmd
+ 3
− 3
View file @ fca96fc6
......@@ -79,9 +79,9 @@ Estimer les tendances de la date potentielle de récolte du maïs grain dans les
Restaurer les packages à utiliser : `renv::restore()`.
Il faut avoir préalablement copié les données depuis le stockage S3 Minio :
Si ce n'est pas déjà fait, il faut avoir préalablement copié les données depuis le stockage MinIO :
`$ mc cp -r s3/projet-funathon/2023/sujet2/era5/ funathon2023_sujet2/donnees/era5/`
`$ mc cp -r s3/projet-funathon/2023/sujet2/diffusion/era5/ funathon2023_sujet2/data/era5/`
Pour se connecter à la base PostgreSQL, il faudra avoir défini votre mot de passe comme variable d'environnement par exemple en ajoutant `PASS_POSTGRESQL=xxxx` dans le fichier *\~/.Renviron* : `file.edit("~/.Renviron")` et relancer la session (ctrl+maj+F10).
......@@ -103,7 +103,7 @@ library(knitr) # génération du document
library(ggrepel) # étiquettage graphiques
# localisation des données dans le stockage "local"
rep_era5 <- "donnees/era5"
rep_era5 <- "data/era5"
# pour avoir les noms de dates en français
invisible(Sys.setlocale("LC_ALL", "fr_FR.UTF-8"))
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment