(file) Return to godi_oasis_import CVS log (file) (dir) Up to [godi-bootstrap] / godi-tools / trunk / console-src / godi_oasis_import

File: [godi-bootstrap] / godi-tools / trunk / console-src / godi_oasis_import (download)
Revision: 1120, Wed Dec 5 16:02:24 2012 UTC (6 years, 4 months ago) by gerd
File size: 19915 byte(s)
Updates for OASIS import

(* -*- tuareg -*- *)

#use "topfind";;
#require "oasis,str,netstring,netclient";;


open Printf
module StrSet = Set.Make(String)

let autoquery_url = ref "http://godirepo.camlcity.org/openapps/autoquery.cgi"

let oasis_url = ref "http://oasis.ocamlcore.org/dev"

let oasis_db_download name version tarball =
  !oasis_url ^ "/dist/" ^ Netencoding.Url.encode ~plus:false name
    ^ "/" ^ Netencoding.Url.encode ~plus:false version ^ "/" ^ 
     Netencoding.Url.encode ~plus:false tarball


let unique l =
  let ht = Hashtbl.create 5 in
  List.filter
    (fun x ->
       if Hashtbl.mem ht x then
         false
       else (
         Hashtbl.add ht x ();
         true
       )
    )
    l

let string_of_channel ch =
  let b = Buffer.create 1000 in
  let data = String.create 16384 in
  let n = ref 1 in
  while !n > 0 do
    n := input ch data 0 16384;
    Buffer.add_substring b data 0 !n
  done;
  Buffer.contents b

let ignore_ch ch =
  let data = String.create 16384 in
  let n = ref 1 in
  while !n > 0 do
    n := input ch data 0 16384;
  done  


let close_process_in cmd ch =
  let s = Unix.close_process_in ch in
  match s with
    | Unix.WEXITED 0 -> ()
    | _ ->
        failwith ("Command fails: " ^ cmd)


let oasis_re = Str.regexp ".*/_oasis$"

let get_oasis_file tarball =
  let cmd =
    sprintf "godi_pax -f %s"
      (Filename.quote tarball) in
  let ch = Unix.open_process_in cmd in
  let found = ref None in
  ( try
      while !found = None do
        let line = input_line ch in
        if Str.string_match oasis_re line 0 then
          found := Some line
      done
    with End_of_file -> ()
  );
  ignore_ch ch;
  close_process_in cmd ch;
  match !found with
    | None ->
        failwith ("Tarball does not contain _oasis file: " ^ tarball)
    | Some oasis_name ->
        let cmd =
          sprintf "godi_pax -r -O -f %s %s"
            (Filename.quote tarball)
            (Filename.quote oasis_name) in
        let ch = Unix.open_process_in cmd in
        let data = string_of_channel ch in
        close_process_in cmd ch;
        (oasis_name,data)

let get url =
  Http_client.Convenience.http_get url

let constraint_fulfilled version version_contraint =
  let open OASISVersion in
  match version_contraint with
    | None -> true
    | Some comp ->
        ( try
            let oasis_v = version_of_string version in
            comparator_apply oasis_v comp
          with
            | _ -> true
        )

let autoquery findlib_name version_constraint =
  let url =
    !autoquery_url ^ 
      "?q=findlib&l=" ^ Netencoding.Url.encode findlib_name in
  let data = get url in
  let ch = new Netchannels.input_string data in
  let by_section = Hashtbl.create 17 in
  ( try
      while true do
        let line = ch # input_line() in
        let (section, pkgname, pkgversion) =
          Scanf.sscanf
            line
            "section=%S platform=%S hostname=%S name=%S version=%S"
            (fun section _ _ name version ->
               (section,name,version)
            ) in
        (* Fixup *)
        let pkgname' =
          if findlib_name <> "findlib" && pkgname = "godi-findlib" then
            "godi-ocaml"
          else
            pkgname in
        if constraint_fulfilled pkgversion version_constraint then
          Hashtbl.replace by_section section pkgname'
          (* FIXME: Take the highest version only. But we'd need a version
             comparator
           *)
      done
    with End_of_file -> ()
  );
  by_section

let autoquery_sub findlib_name version_constraint =
  (* with subpackage resolution *)
  let by_section_1 = autoquery findlib_name version_constraint in
  let by_section_2 =
    if String.contains findlib_name '.' then
      let k = String.index findlib_name '.' in
      let main_name = String.sub findlib_name 0 k in
      autoquery main_name version_constraint
    else
      by_section_1 in
  let by_section = Hashtbl.copy by_section_2 in
  Hashtbl.iter
    (fun section _ ->
       try
         let v = Hashtbl.find by_section_1 section in
         Hashtbl.replace by_section section v
       with Not_found -> ()
    )
    by_section_2;
  by_section


let ht_as_list ht =
  Hashtbl.fold (fun k v acc -> (k,v)::acc) ht []


let ctxt =
  { !OASISContext.default with
      OASISContext.ignore_plugins = true
  }

let parse_oasis s =
  OASISParse.from_string ~ctxt s

let rec extract_bool_choice c =
  let open OASISExpr in
  match c with
    | [] -> EBool false
    | [c1, true] -> c1
    | [c1, false] -> ENot(c1)
    | (c1,true) :: c' -> EOr(c1, extract_bool_choice c')
    | (c1,false) :: c' -> EOr(ENot c1, extract_bool_choice c')


let rec simplif_bool_expr e =
  let open OASISExpr in
  match e with
    | EBool _ 
    | ETest _
    | EFlag _ -> e
    | ENot(EBool b) -> EBool(not b)
    | ENot e1 -> ENot(simplif_bool_expr e1)
    | EAnd(e1,e2) ->
        let e1' = simplif_bool_expr e1 in
        let e2' = simplif_bool_expr e2 in
        if e1' = EBool false || e2' = EBool false then
          EBool false
        else
          if e1' = EBool true then
            e2'
          else if e2' = EBool true then
            e1'
          else
            EAnd(e1',e2')
    | EOr(e1,e2) ->
        let e1' = simplif_bool_expr e1 in
        let e2' = simplif_bool_expr e2 in
        if e1' = EBool true || e2' = EBool true then
          EBool true
        else
          if e1' = EBool false then
            e2'
          else if e2' = EBool false then
            e1'
          else
            EOr(e1',e2')


let rec godi_dep_of_oasis_comparator base comp =
  let open OASISVersion in
  match comp with
    | VGreater v ->
        [ base ^ ">" ^ string_of_version v ]
    | VGreaterEqual v ->
        [ base ^ ">=" ^ string_of_version v ]
    | VEqual v ->
        [ base ^ "==" ^ string_of_version v ]
    | VLesser v ->
        [ base ^ "<" ^ string_of_version v ]
    | VLesserEqual v ->
        [ base ^ "<=" ^ string_of_version v ]
    | VOr _ ->
        []  (* Cannot represent this *)
    | VAnd(comp1,comp2) ->
        godi_dep_of_oasis_comparator base comp1 @
          godi_dep_of_oasis_comparator base comp2


let minus_re = Str.regexp "-"

let conf_variable godi_pkgname flag =
  let v = String.uppercase godi_pkgname ^ "_" ^ flag in
  Str.global_replace minus_re "_" v


let make_expr godi_pkgname e =
  let open OASISExpr in
  let rec transl e =
    match e with
      | EBool true -> "defined(.true)"
      | EBool false -> "defined(.false)"
      | ENot e1 -> "!(" ^ transl e1 ^ ")"
      | EOr(e1,e2) -> "(" ^ transl e1 ^ ") || (" ^ transl e2 ^ ")"
      | EAnd(e1,e2) -> "(" ^ transl e1 ^ ") && (" ^ transl e2 ^ ")"
      | EFlag name ->
          let v = conf_variable godi_pkgname name in
          "defined(" ^  v ^ ")" ^ " && ${" ^ v ^ "} == \"yes\""
      | ETest (t,value) ->
          ( match string_of_test t with
              | "os_type" ->
                  "defined(OCAML_OS_TYPE) && ${OCAML_OS_TYPE} == \""
                  ^ value ^ "\""
              | t_name ->
                  (* system, architecture, cc_type: depend on the result
                     of the OCaml configure script. This is first available
                     when godi-ocaml-src has been built. So we cannot
                     support it
                   *)
                  failwith ("OASIS test '" ^ t_name ^ "' not supported yet")
          ) in
  transl e


let get_provide_findlib oasis_pkg =
  let open OASISTypes in
  let open OASISExpr in

  let ht = Hashtbl.create 10 in

  let rec get_full_name lib_name =
    let (common,lib,nref) = Hashtbl.find ht lib_name in
    match !nref with
      | Some n -> n
      | None ->
          let full_name_prefix =
            match lib.lib_findlib_parent with
              | None -> ""
              | Some parent_lib -> get_full_name parent_lib ^ "." in
          let full_name_middle =
            if lib.lib_findlib_containers = [] then
              ""
            else
              String.concat "." lib.lib_findlib_containers ^ "." in
          let full_name_last =
            match lib.lib_findlib_name with
              | None -> lib_name
              | Some n -> n in
          let full_name = 
            full_name_prefix ^ full_name_middle ^ full_name_last in
          nref := Some full_name;
          full_name
  in

  List.iter
    (function
       | Library(common,_,lib) ->
           Hashtbl.add ht common.cs_name (common,lib,ref None)
       | _ ->
           ()
    )
    oasis_pkg.sections;

  List.flatten
    (List.map
       (function
          | Library(common,build,_) ->
              let cond_build = 
                simplif_bool_expr (extract_bool_choice build.bs_build) in
              let cond_install = 
                simplif_bool_expr (extract_bool_choice build.bs_install) in
              if cond_build <> EBool false && cond_install <> EBool false then
                let lib_name = common.cs_name in
                [ get_full_name lib_name ]
              else
                []
          | _ ->
              []
       )
       oasis_pkg.sections
    )


let get_build_deps oasis_pkg =
  let open OASISTypes in

  let ht = Hashtbl.create 10 in
  List.iter
    (fun name -> Hashtbl.add ht name ())
    (get_provide_findlib oasis_pkg);

  unique
    (List.flatten
       (List.map
          (fun sect ->
             match sect with
               | Library(_, build, _)
               | Executable(_, build, _) ->
                   let cond =
                     simplif_bool_expr
                       (extract_bool_choice build.bs_build) in
                   List.flatten
                     (List.map
                        (fun dep ->
                           match dep with
                             | FindlibPackage(fl,version_constraint) ->
                                 if Hashtbl.mem ht fl then
                                   []  (* internal dep *)
                                 else
                                   [fl, cond, version_constraint]
                             | _ ->
                                 []
                        )
                        build.bs_build_depends
                     )
               | _ ->
                   []
          )
          oasis_pkg.sections
       )
    )


let make_set_from_list l =
  List.fold_left (fun acc s -> StrSet.add s acc) StrSet.empty l

let make_set_from_keys ht =
  Hashtbl.fold (fun s _ acc -> StrSet.add s acc) ht StrSet.empty


let resolve_build_deps deps =
  if deps = [] then
    []
  else
    let deps_plus_pkgs =
      List.map (fun (lib,cond,v) -> lib, cond, autoquery_sub lib v) deps in
    let sections_ht = autoquery "stdlib" None in
    let sections = 
      StrSet.remove "oasis-import" (make_set_from_keys sections_ht) in
    List.flatten
      (List.map
         (fun section ->
            try
              let l =
                List.map
                  (fun (lib,cond,ht) ->
                     let pkgname = 
                       try Hashtbl.find ht section
                       with Not_found ->
                         Hashtbl.find ht "oasis-import" in
                     (lib,pkgname,cond)
                  )
                  deps_plus_pkgs in
              [ section,l ]
            with
              | Not_found -> []
         )
         (StrSet.elements sections)
      )


let list_of_opt =
  function
    | None -> []
    | Some x -> [x]


let write_godi_package tarball oasis_path oasis_pkg oasis_data ord rev dir =
  let open OASISTypes in
  let open OASISExpr in

  let deps = get_build_deps oasis_pkg in
  let rdeps = resolve_build_deps deps in

  let f_descr =
    open_out (dir ^ "/DESCR") in
  ( match oasis_pkg.description with
      | None -> ()
      | Some text ->
          output_string f_descr text;
          output_string f_descr "\n\n"
  );
  fprintf f_descr "*** CAVEAT ***\n\n";
  fprintf f_descr "This package has been converted from OASIS-DB using\n";
  fprintf f_descr "godi_oasis_import. The quality of this package has\n";
  fprintf f_descr "never been checked by GODI packagers.\n";
  close_out f_descr;

  let f_make =
    open_out (dir ^ "/Makefile") in
  fprintf f_make ".include \"../../mk/bsd.prefs.mk\"\n";

  let godi_name = "oasis-" ^ oasis_pkg.name in
  let cap_godi_name =
    Str.global_replace (Str.regexp "-") "_" (String.uppercase godi_name) in
  fprintf f_make "PKGNAME = %s-${VERSION}\n" godi_name;

  let oasis_version =
    OASISVersion.string_of_version oasis_pkg.version in
  let godi_version0 = 
    oasis_version ^ "_ord" ^ string_of_int ord in
  let godi_version =
    Str.global_replace (Str.regexp "-") "_" godi_version0 in
  fprintf f_make "VERSION = %s\n" godi_version;
  fprintf f_make "PKGREVISION = %d\n" rev;

  fprintf f_make "DISTNAME = %s\n" (Filename.dirname oasis_path);
  fprintf f_make "DISTFILES = %s\n" (Filename.basename tarball);
  fprintf f_make "DISTPREFIX = oasis-\n";
  fprintf f_make "CATEGORIES = oasis\n";
  fprintf f_make "MASTER_SITES = %s\n"
    (oasis_db_download oasis_pkg.name oasis_version "");
  fprintf f_make "MAINTAINER = none (auto-imported)\n";
  fprintf f_make "HOMEPAGE = %s\n"
    (match oasis_pkg.homepage with
       | None -> "none"
       | Some url -> url
    );
  fprintf f_make "COMMENT = %s\n"
    (Str.global_replace (Str.regexp "[ \t\n\r]+") " " oasis_pkg.synopsis);

  (* Get names of installed findlibs *)
  (* FIXME: we don't interpret conditionals here, but we could *)
  let provide_findlib = get_provide_findlib oasis_pkg in
  fprintf f_make "PROVIDE_FINDLIB = %s\n" (String.concat " " provide_findlib);
  
  let flags = ref [] in
  List.iter
    (fun sect ->
       match sect with
         | Flag(common,flag) ->
             let flag_name = common.cs_name in
             let var_name = conf_variable godi_name flag_name in
             let cond =
               simplif_bool_expr(extract_bool_choice flag.flag_default) in
             ( match cond with
                 | EBool true ->
                     fprintf f_make "%s ?= yes\n" var_name
                 | EBool false ->
                     fprintf f_make "%s ?= no\n" var_name
                 | _ ->
                     let godi_expr = make_expr godi_name cond in
                     fprintf f_make ".if %s\n" godi_expr;
                     fprintf f_make "%s ?= yes\n" var_name;
                     fprintf f_make ".else\n";
                     fprintf f_make "%s ?= no\n" var_name;
                     fprintf f_make ".endif\n"
             );
             flags := (flag_name, var_name, flag.flag_description) :: !flags
         | _ -> ()
    )
    oasis_pkg.sections;
  flags := List.rev !flags;

  List.iter
    (fun (flag_name, var_name, _) ->
       fprintf f_make ".if ${%s} == \"yes\"\n" var_name;
       fprintf f_make "CONFIGURE_ARGS += --enable-%s\n" flag_name;
       fprintf f_make ".else\n";
       fprintf f_make "CONFIGURE_ARGS += --disable-%s\n" flag_name;
       fprintf f_make ".endif\n";
    )
    !flags;
  fprintf f_make ".ifdef %s_CONFIGURE_ARGS\n" cap_godi_name;
  fprintf f_make "CONFIGURE_ARGS += ${%s_CONFIGURE_ARGS}\n" cap_godi_name;
  fprintf f_make ".endif\n";


  fprintf f_make "BUILD_DEPENDS += godi-tools>=3.0.19\n"; (* FIXME *)

  let findlib_version_l =
    match oasis_pkg.findlib_version with
      | None -> [ "godi-findlib>=1.3" ]
      | Some comp -> godi_dep_of_oasis_comparator "godi-findlib" comp in
  List.iter
    (fun dep ->
       fprintf f_make "BUILD_DEPENDS += %s\n" dep
    )
    findlib_version_l;

  fprintf f_make ".ifdef %s_DEPENDS_OVERRIDE\n" cap_godi_name;
  fprintf f_make "DEPENDS += ${%s_DEPENDS_OVERRIDE}\n" cap_godi_name;
  fprintf f_make ".else\n";

  let ocaml_version_l =
    match oasis_pkg.ocaml_version with
      | None -> [ "godi-ocaml>=0" ]
      | Some comp -> godi_dep_of_oasis_comparator "godi-ocaml" comp in
  List.iter
    (fun dep ->
       fprintf f_make "DEPENDS += %s\n" dep
    )
    ocaml_version_l;

  let init_depends = [ "godi-ocaml" ] in

  let best_section = 
    List.fold_left
      ( max )
      ""
      (List.map fst rdeps) in
  let rdeps_best =
    try [ "", List.assoc best_section rdeps ] with Not_found -> [] in

  let first = ref true in
  List.iter
    (fun (section, deps) ->
       let set = ref(make_set_from_list init_depends) in
       if section = "" then
         fprintf f_make ".else\n"
       else
         fprintf f_make ".%sif ${GODI_SECTION} == %S\n" 
           (if !first then "" else "el") section;
       List.iter
         (fun (lib,pkgname,cond) ->
            if not(StrSet.mem pkgname !set) then (
              set := StrSet.add pkgname !set;
              match cond with
                | EBool true ->
                    fprintf f_make "DEPENDS += %s>=0\n" pkgname
                | EBool false ->
                    ()
                | _ ->
                    let godi_expr = make_expr godi_name cond in
                    fprintf f_make ".if %s\n" godi_expr;
                    fprintf f_make "DEPENDS += %s>=0\n" pkgname;
                    fprintf f_make ".endif\n"
            )
         )
         deps;
       first := false;
    )
    (rdeps @ rdeps_best);
  if not !first then
    fprintf f_make ".endif\n";
  fprintf f_make "%s_DEPENDS_OVERRIDE = ${DEPENDS}\n" cap_godi_name;
  fprintf f_make ".endif\n"; (* .ifdef OVERRIDE *)
  
  fprintf f_make "MAKE_ENV += ${BUILD_OCAMLFIND_ENV}\n"; (* CHECK *)
  fprintf f_make "OASIS = yes\n"; (* TODO *)
  fprintf f_make ".include \"../../mk/bsd.pkg.mk\"\n\n";

  fprintf f_make "# GENERATED FROM THIS _oasis FILE:\n";
  fprintf f_make "# %s\n"
    (Str.global_replace (Str.regexp "\n") "\n# " oasis_data);

  close_out f_make;

  let f_conf = open_out (dir ^ "/CONFOPTS") in
  List.iter
    (fun (flagname,varname,descr) ->
       fprintf f_conf "%s\n" varname
    )
    !flags;
  fprintf f_conf "%s_DEPENDS_OVERRIDE\n" cap_godi_name;
  fprintf f_conf "%s_CONFIGURE_ARGS\n" cap_godi_name;
  close_out f_conf;

  let f_msg = open_out (dir ^ "/BUILDMSG") in
  fprintf f_msg "Available options for godi.conf:\n\n";
  List.iter
    (fun (flagname,varname,descr) ->
       fprintf f_msg "- %s (yes/no):\n" varname;
       match descr with
         | None ->
             fprintf f_msg "  (no description)\n\n";
         | Some text ->
             fprintf f_msg "  %s\n\n" text;
    )
    !flags;
  fprintf f_msg "- %s_DEPENDS_OVERRIDE:\n" cap_godi_name;
  fprintf f_msg "  Override the auto-generated list of dependencies\n\n";
  fprintf f_msg "- %s_CONFIGURE_ARGS:\n" cap_godi_name;
  fprintf f_msg "  Additional args for configuration (ocaml setup.ml -configure)\n\n";
  close_out f_msg


let () =
  let tarball = ref None in
  let ord = ref 0 in
  let rev = ref 0 in
  let dir = ref "." in

  Arg.parse
    [ "-tarball", Arg.String (fun s -> tarball := Some s),
      "<file>   Parse this OASIS-conforming tarball";

      "-ord", Arg.Set_int ord,
      "<n>      Set the OASIS ordinal number";

      "-rev", Arg.Set_int rev,
      "<n>      Set the GODI revision";

      "-dir", Arg.Set_string dir,
      "<dir>    Set the output directory (must exist)";

      "-autoquery", Arg.Set_string autoquery_url,
      "<url>    Set the URL where the autoquery service is available";

      "-oasis", Arg.Set_string oasis_url,
      "<url>    Set the URL where OASIS-DB is available (start page)";
    ]
    (fun arg -> raise(Arg.Bad(arg ^ ": unexpected")))
    "usage: godi_oasis_import -tarball <file.tar.gz>";

  let tarball =
    match !tarball with
      | Some f -> f
      | None -> failwith "Missing -tarball" in

  let oasis_path, oasis_data = get_oasis_file tarball in
  let oasis_pkg = parse_oasis oasis_data in
  write_godi_package tarball oasis_path oasis_pkg oasis_data !ord !rev !dir;

  printf "Written files to \"%s\".\n" !dir;
  printf "Note that a distinfo file has not been created. Run\n";
  printf "  godi_make fetch && godi_make makesum\n";
  printf "to do so.\n"

  

SVN admin
Powered by
ViewCVS 1.0-dev