Rectangling onboarding

library("ghql")
library("httr")
library("magrittr")
# function to get number of last issue
get_last_issue <- function(){
  query = '{
  repository(owner: "ropensci", name: "onboarding") {
  issues(last: 1) {
  edges{
  node{
  number
  }
  }
  }
}
}'
token <- Sys.getenv("GITHUB_GRAPHQL_TOKEN")
  cli <- GraphqlClient$new(
    url = "https://api.github.com/graphql",
    headers = add_headers(Authorization = paste0("Bearer ", token))
  )
  
  ## define query
  ### creat a query class first
  qry <- Query$new()
  qry$query('issues', query)
  last_issue <-cli$exec(qry$queries$issues)
  last_issue %>%
    jqr::jq('.data.repository.issues.edges[].node.number') %>%
   as.numeric()
}
get_last_issue()

#get all threads
issues <- purrr::map_df(1:get_last_issue(), get_issue_thread)

# for the one(s) with 101 comments get the 100 last comments
long_issues <- issues %>%
  dplyr::count(issue) %>%
  dplyr::filter(n == 101) %>%
  dplyr::pull(issue)

issues2 <- purrr::map_df(long_issues, get_issue_thread, first = FALSE)
all_issues <- dplyr::bind_rows(issues, issues2)
all_issues <- unique(all_issues)


readr::write_csv(all_issues, "data/all_threads_v4.csv")

issues <- readr::read_csv("data/all_threads_v4.csv")
issues <- janitor::clean_names(issues)
issues <- dplyr::rename(issues, user = author)
issues <- dplyr::select(issues, - dplyr::contains("topic"))
issues %>%
  head() %>%
  dplyr::select(- body) %>%
  knitr::kable()

# labels
replace_1 <- function(x){
 !is.na(x[1])
}

# binary variables
ncol_issues <- ncol(issues)
issues <- dplyr::group_by(issues, issue) %>%
  dplyr::arrange(created_at) %>%
  dplyr::mutate_at(9:(ncol_issues-1), replace_1) %>%
  dplyr::ungroup()


# keep only issues that are finished
issues <- dplyr::filter(issues, package, !x0_presubmission, 
                        !out_of_scope, !legacy,
                        !x1_editor_checks, x6_approved)
issues <- dplyr::select(issues, - dplyr::starts_with("x"),
                        - package, - out_of_scope, - legacy,
                        - meta, - holding, - pulled, - question)

# airtable data
airtable <- airtabler::airtable("appZIB8hgtvjoV99D", "Reviews")
airtable <- airtable$Reviews$select_all()

airtable <- dplyr::mutate(airtable,
                          issue = as.numeric(stringr::str_replace(onboarding_url, 
                                                                  ".*issues\\/", "")))

# we get the name of the package
# and we know which comments are reviews
reviews <- dplyr::select(airtable, review_url, issue, package) %>%
  dplyr::mutate(is_review = TRUE) 

issues <- dplyr::left_join(issues, reviews, by = c("issue", "comment_url" = "review_url"))
issues <- dplyr::mutate(issues, is_review = !is.na(is_review))

# non elegant code to guess role
issues <- dplyr::group_by(issues, issue)
issues <- dplyr::arrange(issues, created_at)
issues <- dplyr::mutate(issues, author = user[1])
issues <- dplyr::mutate(issues, package = unique(package[!is.na(package)]))
issues <- dplyr::mutate(issues, assignee = assignee[1])
issues <- dplyr::mutate(issues, reviewer1 = ifelse(!is.na(user[is_review][1]), user[is_review][1], ""))
issues <- dplyr::mutate(issues, reviewer2 = ifelse(!is.na(user[is_review][2]), user[is_review][2], ""))
issues <- dplyr::mutate(issues, reviewer3 = ifelse(!is.na(user[is_review][3]), user[is_review][3], ""))
issues <- dplyr::ungroup(issues)
issues <- dplyr::group_by(issues, issue, created_at, user)
# regexp because in at least 1 case assignee = 2 names glued together
issues <- dplyr::mutate(issues, commenter = ifelse(stringr::str_detect(assignee, user), "editor", "other"))
issues <- dplyr::mutate(issues, commenter = ifelse(user == author, "author", commenter))
issues <- dplyr::mutate(issues, commenter = ifelse(user == reviewer1, "reviewer1", commenter))
issues <- dplyr::mutate(issues, commenter = ifelse(user == reviewer2, "reviewer2", commenter))
issues <- dplyr::mutate(issues, commenter = ifelse(user == reviewer3, "reviewer3", commenter))
issues <- dplyr::mutate(issues, commenter = ifelse(user == "stefaniebutland", "community_manager", commenter))
issues <- dplyr::ungroup(issues)
issues <- dplyr::mutate(issues, role = commenter,
                        role = ifelse(stringr::str_detect(role, "reviewer"),
                                      "reviewer", role))

issues <- dplyr::select(issues, - author, - reviewer1, - reviewer2, - reviewer3, - assignee,
                        - author_association, - comment_url)
readr::write_csv(issues, "data/clean_data.csv")

issues %>%
  dplyr::select(- body) %>%
  head() %>%
  knitr::kable()

airtable <- airtabler::airtable("appZIB8hgtvjoV99D", "Reviews")
airtable <- airtable$Reviews$select_all()

safe_clone <- purrr::safely(git2r::clone)

# github link either ropensci or ropenscilabs
clone_repo <- function(package_name){
  print(package_name)
  url <- paste0("https://github.com/ropensci/", package_name, ".git")
  local_path <- paste0(getwd(), "/repos/", package_name)
  clone_from_ropensci <- safe_clone(url = url, local_path = local_path,
                                    progress = FALSE)
  if(is.null(clone_from_ropensci$result)){
    url <- paste0("https://github.com/ropenscilabs/", package_name, ".git")
    clone_from_ropenscilabs <- safe_clone(url = url, local_path = local_path,
                                          progress = FALSE)
    if(is.null(clone_from_ropenscilabs$result)){
      message("OUILLE")
    }
  }
  
}

pkgs <- unique(airtable$package)
pkgs <- pkgs[!pkgs %in% fs::dir_ls()]
pkgs <- pkgs[pkgs != "rrricanes"]
purrr::walk(pkgs, clone_repo)

library("magrittr")

get_report <- function(package_name){
  message(package_name)
  local_path <- paste0(getwd(), "/repos/", package_name)
  if(length(fs::dir_ls(local_path)) != 0){
    gitsum::init_gitsum(local_path, over_write = TRUE)
    report <- gitsum::parse_log_detailed(local_path)
    report <- dplyr::select(report, - nested)
    report$package <- package_name
    if(!"datetime" %in% names(report)){
      report <-  dplyr::mutate(report,
                               hour = as.numeric(stringr::str_sub(timezone, 1, 3)),
                               minute = as.numeric(stringr::str_sub(timezone, 4, 5)),
                               datetime = date + lubridate::hours(-1 * hour) + lubridate::minutes(-1 * minute))
     report <- dplyr::select(report, - hour, - minute, - timezone)
      
    }
    report <- dplyr::select(report, - date)
    return(report)
  }else{
    return(NULL)
  }
}

packages <- fs::dir_ls("repos") 
packages <- stringr::str_replace_all(packages, "repos\\/", "")
purrr::map_df(packages, get_report) %>%
   readr::write_csv("output/gitsum_reports.csv")

library("magrittr")
# get issues opening datetime
issues <- readr::read_csv("data/clean_data.csv")
issues <- dplyr::group_by(issues, package)
issues <- dplyr::summarise(issues, opened = min(created_at))

# now for each package keep only commits before that
commits <- readr::read_csv("output/gitsum_reports.csv")
commits <- dplyr::left_join(commits, issues, by = "package")
commits <- dplyr::group_by(commits, package)
commits <- dplyr::filter(commits, datetime <= opened)
# and from them keep the latest one, 
# that's the latest commit before submission!
commits <- dplyr::filter(commits, datetime == max(datetime), !is_merge)
commits <- dplyr::summarize(commits, hash = hash[1])

# small helper function
get_sha <- function(commit){
  commit@sha
}

set_archive <- function(package_name, commit){
  message(package_name)
  # copy the entire repo to another location
  local_path <- paste0(getwd(), "/repos/", package_name)
  local_path_archive <- paste0(getwd(), "/repos_at_submission/", package_name)
  fs::dir_copy(local_path, local_path_archive)
  
  # get all commits -- it's fast which is why I don't use gitsum report here
  commits <- git2r::commits(git2r::repository(local_path_archive))
  
  # get their sha
  sha <- purrr::map_chr(commits, get_sha)
  
  # all of this to extract the commit with the sha of the latest commit before submission
  # in other words the latest commit before submission
  commit <- commits[sha == commit][[1]]
  
  # do a hard reset at that commit
  git2r::reset(commit, reset_type = "hard")
}

purrr::walk2(commits$package, commits$hash, set_archive)