TF-IDF d'un fichier txt
informaticienne
-
KX Messages postés 19031 Statut Modérateur -
KX Messages postés 19031 Statut Modérateur -
Bonjour
je veux implémenter au java le TF-IDf d'un fichier txt mais je ne sais pas comment je vais appellé le fichier txt au main pour calculez son TF-IDf
this code just un example simple de l'excution de TF-IDF d'un liste des mots alors svp dit moi comment je peux calculez le TF-IDF D'un fichier text
merçi pour tt
je veux implémenter au java le TF-IDf d'un fichier txt mais je ne sais pas comment je vais appellé le fichier txt au main pour calculez son TF-IDf
import java.util.Arrays;
import java.util.List;
public class exer {
private static tfidfCalcultor calculator;
/**
* @param doc list of strings
* @param term String represents a term
* @return term frequency of term in document
*/
public static double tf(List<String> doc, String term) {
double result = 0;
for (String word : doc) {
if (term.equalsIgnoreCase(word))
result++;
}
return result / doc.size();
}
/**
* @param docs list of list of strings represents the dataset
* @param term String represents a term
* @return the inverse term frequency of term in documents
*/
public static double idf(List<List<String>> docs, String term) {
double n = 0;
for (List<String> doc : docs) {
for (String word : doc) {
if (term.equalsIgnoreCase(word)) {
n++;
break;
}
}
}
return Math.log(docs.size() / n);
}
/**
* @param doc a text document
* @param docs all documents
* @param term term
* @return the TF-IDF of term
*/
public static double tfIdf(List<String> doc, List<List<String>> docs, String term) {
return tf(doc, term) * idf(docs, term);
}
public static void main(String[] args) {
List<String> doc1 = Arrays.asList("Lorem", "ipsum", "dolor", "ipsum", "sit", "ipsum");
List<String> doc2 = Arrays.asList("Vituperata", "incorrupte", "at", "ipsum", "pro", "quo");
List<String> doc3 = Arrays.asList("Has", "persius", "disputationi", "id", "simul", "lorem");
List<List<String>> documents = Arrays.asList(doc1, doc2, doc3);
calculator = new tfidfCalcultor();
double tfidf = tfIdf(doc1, documents, "lorem");
System.out.println("TF-IDF (lorem) = " + tfidf);
}
}
this code just un example simple de l'excution de TF-IDF d'un liste des mots alors svp dit moi comment je peux calculez le TF-IDF D'un fichier text
merçi pour tt
1 réponse
-
Bonjour,
Si tu sais le faire pour une liste de mots, il suffit juste de lire un fichier texte, créer la liste de mots qu'il contient, et travailler sur la liste...
List<String> list = new ArrayList<>(); Scanner sc = new Scanner(new FileInputStream("C:\\test.txt")); while (sc.hasNext()) list.add(sc.next()); sc.close(); System.out.println(list);-
-
-
-
Dans le code de départ tu as :
public static void main(String[] args) { List<String> doc1 = Arrays.asList("Lorem", "ipsum", "dolor", "ipsum", "sit", "ipsum"); List<String> doc2 = Arrays.asList("Vituperata", "incorrupte", "at", "ipsum", "pro", "quo"); List<String> doc3 = Arrays.asList("Has", "persius", "disputationi", "id", "simul"); List<List<String>> documents = Arrays.asList(doc1, doc2, doc3);
Si tu remplaces les listes par celles issues de fichiers (avec mon code) ça donne :public static void main(String[] args) { List<String> doc1 = new ArrayList<>(); Scanner sc = new Scanner(new FileInputStream("C:\\doc1.txt")); while (sc.hasNext()) doc1.add(sc.next()); sc.close(); // idem pour doc2, doc3... List<List<String>> documents = Arrays.asList(doc1, doc2, doc3); -
import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Scanner; public class TF { private static tfidfCalcultor calculator; /** * @param doc list of strings * @param term String represents a term * @return term frequency of term in document */ public static double tf(List<String> doc, String term) { double result = 0; for (String word : doc) { if (term.equalsIgnoreCase(word)) result++; } return result / doc.size(); } /** * @param docs list of list of strings represents the dataset * @param term String represents a term * @return the inverse term frequency of term in documents */ public static double idf(List<List<String>> docs, String term) { double n = 0; for (List<String> doc : docs) { for (String word : doc) { if (term.equalsIgnoreCase(word)) { n++; break; } } } return Math.log(docs.size() / n); } /** * @param doc a text document * @param docs all documents * @param term term * @return the TF-IDF of term */ public static double tfIdf(List<String> doc, List<List<String>> docs, String term) { return tf(doc, term) * idf(docs, term); } public static void main(String[] args) throws IOException { try { FileReader fr = new FileReader(new File("e.txt")); BufferedReader br = new BufferedReader(fr); String str; while ((str = br.readLine()) != null) { System.out.println(str + "\n"); } br.close(); } catch (IOException e) { System.out.println("file not found "); } calculator = new tfidfCalcultor(); List<String> list = new ArrayList<>(); Scanner sc = new Scanner(new FileInputStream("e.txt")); while (sc.hasNext()) list.add(sc.next()); sc.close(); System.out.println(list); List<List<String>> documents = Arrays.asList(list); double tfidf = tfIdf(list, documents, "e.txt"); System.out.println("TF-IDF (fichier) = " + tfidf); } }
mais le tfidf ça marche pas
le code ça affiche :
helloo every body
are you okay
fine men
[helloo, every, body, are, you, okay, fine, men]
TF-IDF (fichier) = NaN
-
